LIVY-150. Use "livy-server start|stop" in RealCluster ITs.

This makes sure those code paths are tested too. While at it, I did some
small clean up in the server script, and added documentation for some more
env variables.

Closes #136

bin/livy-server

@@ -21,16 +21,6 @@
 
 usage="Usage: livy-server (start|stop)"
 
-# Find the java binary
-if [ -n "${JAVA_HOME}" ]; then
-  RUNNER="${JAVA_HOME}/bin/java"
-elif [ `command -v java` ]; then
-  RUNNER="java"
-else
-  echo "JAVA_HOME is not set" >&2
-  exit 1
-fi
-
 export LIVY_HOME=$(cd $(dirname $0)/.. && pwd)
 LIVY_CONF_DIR=${LIVY_CONF_DIR:-"$LIVY_HOME/conf"}
 
@@ -41,13 +31,24 @@ if [ -f "${LIVY_CONF_DIR}/livy-env.sh" ]; then
   set +a
 fi
 
+# Find the java binary
+if [ -n "${JAVA_HOME}" ]; then
+  RUNNER="${JAVA_HOME}/bin/java"
+elif [ `command -v java` ]; then
+  RUNNER="java"
+else
+  echo "JAVA_HOME is not set" >&2
+  exit 1
+fi
+
 LIVY_IDENT_STRING=${LIVY_IDENT_STRING:-"$USER"}
 LIVY_PID_DIR=${LIVY_PID_DIR:-"/tmp"}
+LIVY_MAX_LOG_FILES=${LIVY_MAX_LOG_FILES:-5}
 pid="$LIVY_PID_DIR/livy-$LIVY_IDENT_STRING-server.pid"
 
-livy_rotate_log () {
+livy_rotate_log() {
   log=$1
-  num=${2:-5}
+  num=$LIVY_MAX_LOG_FILES
   if [ -f "$log" ]; then # rotate logs
 	while [ $num -gt 1 ]; do
 	  prev=`expr $num - 1`
@@ -61,19 +62,16 @@ livy_rotate_log () {
 create_dir() {
   dir_name=$1
   dir_variable=$2
-  mkdir -p $dir_name
-  touch $dir_name/.test > /dev/null 2>&1
-  TEST_DIR=$?
-  if [ "${TEST_DIR}" = "0" ]; then
-    rm -f "$dir_name"/.test
-  else
-    echo "$USER doesn't have permission to write to directory: $dir_name which is $dir_variable"
+  if [ ! -d "$dir_name" ]; then
+    mkdir -p $dir_name
+  fi
+  if [ ! -w "$dir_name" ]; then
+    echo "$USER doesn't have permission to write to directory $dir_name (defined by $dir_variable)."
     exit 1
   fi
 }
 
 start_livy_server() {
-
   LIBDIR="$LIVY_HOME/jars"
   if [ ! -d "$LIBDIR" ]; then
     LIBDIR="$LIVY_HOME/server/target/jars"
@@ -145,7 +143,6 @@ case $option in
     ;;
 
   (stop)
-
     if [ -f $pid ]; then
       TARGET_ID="$(cat "$pid")"
       if [[ $(ps -p "$TARGET_ID" -o comm=) =~ "java" ]]; then

conf/livy-env.sh

@@ -16,9 +16,18 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-# LIVY ENVIRONMENT VARIABLE
+# LIVY ENVIRONMENT VARIABLES
 #
-# - SPARK_HOME      Spark which you would like to use in livy
-# - LIVY_LOG_DIR    Where log files are stored.  (Default: ${LIVY_HOME}/logs)
+# - JAVA_HOME       Java runtime to use. By default use "java" from PATH.
+# - HADOOP_CONF_DIR Directory containing the Hadoop / YARN configuration to use.
+# - SPARK_HOME      Spark which you would like to use in Livy.
+# - SPARK_CONF_DIR  Optional directory where the Spark configuration lives.
+#                   (Default: $SPARK_HOME/conf)
+# - LIVY_LOG_DIR    Where log files are stored. (Default: ${LIVY_HOME}/logs)
 # - LIVY_PID_DIR    Where the pid file is stored. (Default: /tmp)
-# - LIVY_SERVER_JAVA_OPTS  Java Opts for running livy server (You can set jvm related setting here, like jvm memory/gc algorithm and etc.)
+# - LIVY_SERVER_JAVA_OPTS  Java Opts for running livy server (You can set jvm related setting here,
+#                          like jvm memory/gc algorithm and etc.)
+# - LIVY_IDENT_STRING A name that identifies the Livy server instance, used to generate log file
+#                     names. (Default: name of the user starting Livy).
+# - LIVY_MAX_LOG_FILES Max number of log file to keep in the log directory. (Default: 5.)
+# - LIVY_NICENESS   Niceness of the Livy server process when running in the background. (Default: 0.)

integration-test/src/main/scala/com/cloudera/livy/test/framework/RealCluster.scala

@@ -19,6 +19,7 @@
 package com.cloudera.livy.test.framework
 
 import java.io.{File, IOException}
+import java.nio.file.Files
 import java.security.PrivilegedExceptionAction
 import javax.servlet.http.HttpServletResponse._
 
@@ -51,6 +52,8 @@ private class RealClusterConfig(config: Map[String, String]) {
   val sparkHome = config("env.spark_home")
   val sparkConf = config.getOrElse("env.spark_conf", "/etc/spark/conf")
   val hadoopConf = config.getOrElse("env.hadoop_conf", "/etc/hadoop/conf")
+
+  val javaHome = config.getOrElse("env.java_home", "/usr/java/default")
 }
 
 class RealCluster(_config: Map[String, String])
@@ -99,7 +102,10 @@ class RealCluster(_config: Map[String, String])
     info(s"Running command: $cmd")
     val result = sshClient(_.exec(cmd))
     result.exitCode match {
-      case Some(ec) if ec > 0 => throw new IOException(s"Command failed: $ec")
+      case Some(ec) if ec > 0 =>
+        throw new IOException(s"Command '$cmd' failed: $ec\n" +
+          s"stdout: ${result.stdOutAsString()}\n" +
+          s"stderr: ${result.stdErrAsString()}\n")
       case _ =>
     }
     result
@@ -216,20 +222,22 @@ class RealCluster(_config: Map[String, String])
     upload(livyConfFile.getAbsolutePath(), s"$livyHomePath/conf/livy.conf")
 
     val env = Map(
+        "JAVA_HOME" -> config.javaHome,
         "HADOOP_CONF_DIR" -> config.hadoopConf,
         "SPARK_CONF_DIR" -> sparkConfDir,
         "SPARK_HOME" -> config.sparkHome,
-        "CLASSPATH" -> config.livyClasspath
+        "CLASSPATH" -> config.livyClasspath,
+        "LIVY_PID_DIR" -> s"$tempDirPath/pid",
+        "LIVY_LOG_DIR" -> s"$tempDirPath/logs",
+        "LIVY_MAX_LOG_FILES" -> "16",
+        "LIVY_IDENT_STRING" -> "it"
       )
-      .map { case (k, v) => s"$k=$v" }
-      .mkString(" ")
-
-    val livyServerPath = s"$livyHomePath/bin/livy-server"
+    val livyEnvFile = File.createTempFile("livy-env.", ".sh")
+    saveProperties(env, livyEnvFile)
+    upload(livyEnvFile.getAbsolutePath(), s"$livyHomePath/conf/livy-env.sh")
 
     info(s"Starting Livy @ port ${config.livyPort}...")
-    livyEpoch += 1
-    val logPath = s"$tempDirPath/livy-$livyEpoch.log"
-    exec(s"env $env nohup $livyServerPath > $logPath 2>&1 &")
+    exec(s"env -i $livyHomePath/bin/livy-server start")
     livyIsRunning = true
 
     val httpClient = new AsyncHttpClient()
@@ -242,7 +250,7 @@ class RealCluster(_config: Map[String, String])
   override def stopLivy(): Unit = synchronized {
     info("Stopping Livy Server")
     try {
-      exec(s"pkill -f com.cloudera.livy.server.LivyServer")
+      exec(s"$livyHomePath/bin/livy-server stop")
     } catch {
       case e: Exception =>
         if (livyIsRunning) {
@@ -254,9 +262,11 @@ class RealCluster(_config: Map[String, String])
       // Wait a tiny bit so that the process finishes closing its output files.
       Thread.sleep(2)
 
-      val logName = s"livy-$livyEpoch.log"
-      val logPath = s"$tempDirPath/$logName"
-      val localLog = sys.props("java.io.tmpdir") + File.separator + logName
+      livyEpoch += 1
+      val logName = "livy-it-server.out"
+      val localName = s"livy-it-server-$livyEpoch.out"
+      val logPath = s"$tempDirPath/logs/$logName"
+      val localLog = sys.props("java.io.tmpdir") + File.separator + localName
       download(logPath, localLog)
       info(s"Log for epoch $livyEpoch available at $localLog")
     }