livy/integration-test/pom.xml

<?xml version="1.0" encoding="UTF-8"?>
<!--
  ~ Licensed to the Apache Software Foundation (ASF) under one or more
  ~ contributor license agreements.  See the NOTICE file distributed with
  ~ this work for additional information regarding copyright ownership.
  ~ The ASF licenses this file to You under the Apache License, Version 2.0
  ~ (the "License"); you may not use this file except in compliance with
  ~ the License.  You may obtain a copy of the License at
  ~
  ~    http://www.apache.org/licenses/LICENSE-2.0
  ~
  ~ Unless required by applicable law or agreed to in writing, software
  ~ distributed under the License is distributed on an "AS IS" BASIS,
  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  ~ See the License for the specific language governing permissions and
  ~ limitations under the License.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
  <modelVersion>4.0.0</modelVersion>
  <parent>
    <groupId>org.apache.livy</groupId>
    <artifactId>livy-main</artifactId>
    <relativePath>../pom.xml</relativePath>
    <version>0.8.0-incubating-SNAPSHOT</version>
  </parent>

  <artifactId>livy-integration-test</artifactId>
  <version>0.8.0-incubating-SNAPSHOT</version>
  <packaging>jar</packaging>

  <properties>
    <!--
      Set this to the file containing the cluster description to use. By default a mini cluster
      with 1 DN and 1 NM will be run. Location is tried first in the classpath, then the file
      system.
    -->
    <cluster.spec>default</cluster.spec>
    <skipDeploy>true</skipDeploy>
    <livy.test.thrift.enabled>false</livy.test.thrift.enabled>
  </properties>

  <dependencies>

    <dependency>
      <groupId>${project.groupId}</groupId>
      <artifactId>livy-core_${scala.binary.version}</artifactId>
      <version>${project.version}</version>
    </dependency>

    <dependency>
      <groupId>${project.groupId}</groupId>
      <artifactId>livy-server</artifactId>
      <version>${project.version}</version>
    </dependency>

    <dependency>
      <groupId>${project.groupId}</groupId>
      <artifactId>livy-client-http</artifactId>
      <version>${project.version}</version>
      <scope>test</scope>
    </dependency>

    <dependency>
      <groupId>${project.groupId}</groupId>
      <artifactId>livy-test-lib</artifactId>
      <version>${project.version}</version>
      <scope>test</scope>
    </dependency>

    <dependency>
      <groupId>com.fasterxml.jackson.core</groupId>
      <artifactId>jackson-core</artifactId>
    </dependency>

    <dependency>
      <groupId>com.fasterxml.jackson.core</groupId>
      <artifactId>jackson-databind</artifactId>
    </dependency>

    <dependency>
      <groupId>com.fasterxml.jackson.module</groupId>
      <artifactId>jackson-module-scala_${scala.binary.version}</artifactId>
    </dependency>

    <dependency>
      <groupId>org.asynchttpclient</groupId>
      <artifactId>async-http-client</artifactId>
    </dependency>

    <dependency>
      <groupId>commons-io</groupId>
      <artifactId>commons-io</artifactId>
      <version>2.5</version>
    </dependency>

    <dependency>
      <groupId>org.apache.hadoop</groupId>
      <artifactId>hadoop-common</artifactId>
      <classifier>tests</classifier>
      <exclusions>
        <exclusion>
          <groupId>org.apache.curator</groupId>
          <artifactId>curator-client</artifactId>
        </exclusion>
        <exclusion>
          <groupId>org.apache.zookeeper</groupId>
          <artifactId>zookeeper</artifactId>
        </exclusion>
      </exclusions>
    </dependency>

    <dependency>
      <groupId>org.apache.hadoop</groupId>
      <artifactId>hadoop-hdfs</artifactId>
      <classifier>tests</classifier>
    </dependency>

    <dependency>
      <groupId>org.apache.hadoop</groupId>
      <artifactId>hadoop-yarn-client</artifactId>
    </dependency>

    <dependency>
      <groupId>org.apache.hadoop</groupId>
      <artifactId>hadoop-client</artifactId>
      <exclusions>
        <exclusion>
          <groupId>javax.servlet</groupId>
          <artifactId>servlet-api</artifactId>
        </exclusion>
      </exclusions>
    </dependency>

    <dependency>
      <groupId>org.apache.hadoop</groupId>
      <artifactId>hadoop-yarn-server-tests</artifactId>
      <classifier>tests</classifier>
      <exclusions>
        <exclusion>
          <groupId>org.apache.zookeeper</groupId>
          <artifactId>zookeeper</artifactId>
        </exclusion>
      </exclusions>

    </dependency>

    <dependency>
      <groupId>org.apache.spark</groupId>
      <artifactId>spark-core_${scala.binary.version}</artifactId>
      <scope>test</scope>
    </dependency>

    <dependency>
      <groupId>org.apache.spark</groupId>
      <artifactId>spark-sql_${scala.binary.version}</artifactId>
      <scope>test</scope>
    </dependency>

    <dependency>
      <groupId>org.apache.spark</groupId>
      <artifactId>spark-streaming_${scala.binary.version}</artifactId>
      <scope>test</scope>
    </dependency>

    <dependency>
      <groupId>org.apache.spark</groupId>
      <artifactId>spark-hive_${scala.binary.version}</artifactId>
      <scope>test</scope>
    </dependency>

    <dependency>
      <groupId>org.scala-lang</groupId>
      <artifactId>scala-library</artifactId>
    </dependency>

    <dependency>
      <groupId>org.scalatest</groupId>
      <artifactId>scalatest_${scala.binary.version}</artifactId>
      <version>${scalatest.version}</version>
      <scope>compile</scope>
    </dependency>

    <dependency>
      <groupId>org.scalatra</groupId>
      <artifactId>scalatra-test_${scala.binary.version}</artifactId>
    </dependency>

    <!-- hadoop-common in test scope needs curator-client, but we want to use the proper version -->
    <dependency>
      <groupId>org.apache.curator</groupId>
      <artifactId>curator-client</artifactId>
      <version>${curator.version}</version>
      <exclusions>
        <exclusion>
          <groupId>org.apache.zookeeper</groupId>
          <artifactId>zookeeper</artifactId>
        </exclusion>
      </exclusions>
    </dependency>

    <!-- hadoop-common, hadoop-yarn-server-tests and curator needs zookeeper, but we want to use the proper version -->
    <dependency>
      <groupId>org.apache.zookeeper</groupId>
      <artifactId>zookeeper</artifactId>
      <version>${zookeeper.version}</version>
    </dependency>

  </dependencies>

  <build>
    <plugins>
      <plugin>
        <groupId>org.apache.maven.plugins</groupId>
        <artifactId>maven-antrun-plugin</artifactId>
        <version>1.6</version>
        <executions>
          <execution>
            <id>unpack-spark-files</id>
            <goals>
              <goal>run</goal>
            </goals>
            <phase>test-compile</phase>
            <configuration>
              <tasks>
                <echo message="unpack spark files" />
<!--                <untar src="../spark-2.4.7-bin-hadoop2.7.tgz" compression="gzip" dest="${project.build.directory}" />-->
                <untar src="../spark-3.0.3-bin-hadoop3.2.tgz" compression="gzip" dest="${project.build.directory}" />
              </tasks>
            </configuration>
          </execution>
        </executions>
      </plugin>

      <plugin>
        <groupId>org.codehaus.mojo</groupId>
        <artifactId>build-helper-maven-plugin</artifactId>
        <executions>
          <execution>
            <id>parse-spark-version</id>
            <phase>process-test-sources</phase>
            <goals>
              <goal>parse-version</goal>
            </goals>
            <configuration>
              <propertyPrefix>spark</propertyPrefix>
              <versionString>${spark.version}</versionString>
            </configuration>
          </execution>
          <execution>
            <id>add-spark-version-specific-test</id>
            <phase>process-test-sources</phase>
            <goals>
              <goal>add-test-source</goal>
            </goals>
            <configuration>
              <sources>
                <source>${project.basedir}/src/test/spark${spark.majorVersion}/scala</source>
              </sources>
            </configuration>
          </execution>
        </executions>
      </plugin>

      <plugin>
        <groupId>org.apache.maven.plugins</groupId>
        <artifactId>maven-surefire-plugin</artifactId>
        <executions>
          <execution>
            <id>default-test</id>
            <phase>none</phase>
          </execution>
        </executions>
      </plugin>

      <plugin>
        <groupId>org.scalatest</groupId>
        <artifactId>scalatest-maven-plugin</artifactId>
        <executions>
          <!-- Unbind integration test from test phase and bind it to integration-test phase -->
          <execution>
            <id>test</id>
            <phase>none</phase>
          </execution>
          <execution>
            <id>integration-test</id>
            <phase>integration-test</phase>
            <goals>
              <goal>test</goal>
            </goals>
          </execution>
        </executions>
        <configuration>
          <environmentVariables>
            <LIVY_HOME>${execution.root}</LIVY_HOME>
            <LIVY_TEST>false</LIVY_TEST>
            <LIVY_INTEGRATION_TEST>true</LIVY_INTEGRATION_TEST>
            <SPARK_HOME>${project.build.directory}/${spark.bin.name}</SPARK_HOME>
            <LIVY_TEST_THRIFT_ENABLED>${livy.test.thrift.enabled}</LIVY_TEST_THRIFT_ENABLED>
            <PYSPARK_ROW_FIELD_SORTING_ENABLED>true</PYSPARK_ROW_FIELD_SORTING_ENABLED>
          </environmentVariables>
          <systemProperties>
            <cluster.spec>${cluster.spec}</cluster.spec>
          </systemProperties>
          <skipTests>${skipITs}</skipTests>
        </configuration>
      </plugin>
    </plugins>
  </build>

  <profiles>
    <profile>
      <id>skip-integration-test</id>
      <activation>
        <property>
          <name>skipITs</name>
          <value>true</value>
        </property>
      </activation>
    </profile>
    <profile>
      <id>thriftserver</id>

      <properties>
        <livy.test.thrift.enabled>true</livy.test.thrift.enabled>
      </properties>

      <dependencies>

        <dependency>
          <groupId>${project.groupId}</groupId>
          <artifactId>livy-thriftserver</artifactId>
          <version>${project.version}</version>
        </dependency>

        <!-- needed for testing thriftserver -->
        <dependency>
          <groupId>org.apache.hive</groupId>
          <artifactId>hive-jdbc</artifactId>
          <version>${hive.version}</version>
          <scope>test</scope>
          <exclusions>
            <exclusion>
              <groupId>org.apache.zookeeper</groupId>
              <artifactId>zookeeper</artifactId>
            </exclusion>
            <exclusion>
              <groupId>io.netty</groupId>
              <artifactId>netty</artifactId>
            </exclusion>
            <exclusion>
              <groupId>org.apache.logging.log4j</groupId>
              <artifactId>log4j-slf4j-impl</artifactId>
            </exclusion>
            <exclusion>
              <groupId>org.slf4j</groupId>
              <artifactId>slf4j-log4j12</artifactId>
            </exclusion>
            <exclusion>
              <groupId>org.apache.hbase</groupId>
              <artifactId>*</artifactId>
            </exclusion>
          </exclusions>
        </dependency>

      </dependencies>

      <build>
        <plugins>
          <plugin>
            <groupId>org.codehaus.mojo</groupId>
            <artifactId>build-helper-maven-plugin</artifactId>
            <executions>
              <execution>
                <id>add-scala-sources</id>
                <phase>generate-sources</phase>
                <goals>
                  <goal>add-source</goal>
                </goals>
                <configuration>
                  <sources>
                    <source>${project.basedir}/src/thriftserver/main/scala</source>
                  </sources>
                </configuration>
              </execution>
              <execution>
                <id>add-scala-test-sources</id>
                <phase>generate-test-sources</phase>
                <goals>
                  <goal>add-test-source</goal>
                </goals>
                <configuration>
                  <sources>
                    <source>${project.basedir}/src/thriftserver/test/scala</source>
                  </sources>
                </configuration>
              </execution>
            </executions>
          </plugin>
        </plugins>
      </build>
    </profile>
  </profiles>
</project>