[LIVY-141][LIVY-175][DOCS] Update javadocs and scaladocs and include in Docs build

[LIVY-141](https://issues.apache.org/jira/browse/LIVY-141) [LIVY-175](https://issues.apache.org/jira/browse/LIVY-175)

Adding javadocs to Livy:
- Add ability to build Livy javadocs
- Update current javadoc comments to address build errors and warnings
- Add more javadoc comments to fully describe API
- Include public API javadocs and scaladocs in Livy Documentation

Noted Remaining Issues:
- Since only the public javadocs are build with the Livy Docs not all javadoc warnings in other modules have been addressed.
- There are still some warnings in the Scala API scaladocs build due to the docs not linking to an external lib for referenced scala lib classes, this does not break the build in any way.
- Scaladocs is not supported for Livy as a whole. With the update to Scala 2.11 many scaladoc warnings were upgraded to error and Livy fundamentally can't fix them.

Author: Alex Bozarth <ajbozart@us.ibm.com>

Closes #38 from ajbozarth/javadoc.

.gitignore

@@ -14,6 +14,7 @@
 conf/*.conf
 conf/*.properties
 conf/*.sh
+docs/api/
 lib_managed/
 logs/
 src_managed/

api/src/main/java/org/apache/livy/JobContext.java

@@ -33,25 +33,51 @@ import org.apache.spark.streaming.api.java.JavaStreamingContext;
  */
 public interface JobContext {
 
-  /** The shared SparkContext instance. */
+  /**
+   * @return The shared SparkContext instance.
+   */
   JavaSparkContext sc();
 
-  /** The shared SQLContext instance. */
+  /**
+   * @return The shared SQLContext instance.
+   */
   SQLContext sqlctx();
 
-  /** The shared HiveContext instance. */
+  /**
+   * @return The shared HiveContext instance.
+   */
   HiveContext hivectx();
 
-  /** Returns the JavaStreamingContext which has already been created. */
+  /**
+   * @return The JavaStreamingContext which has already been created.
+   */
   JavaStreamingContext streamingctx();
 
-  /** Get shared object */
+  /**
+   * Get shared object
+   *
+   * @param <E> The type of the shared object
+   * @param name Name of the shared object to return
+   * @return The shared object matching name
+   */
   <E> E getSharedObject(String name) throws NoSuchElementException;
 
-  /** Set shared object, it will replace the old one if already existed */
+  /**
+   * Set shared object, it will replace the old one if already existed
+   *
+   * @param <E> The type of the shared object
+   * @param name Name of the shared object to be set
+   * @param object The object to be set
+   */
   <E> void setSharedObject(String name, E object);
 
-  /** Remove shared object from cache */
+  /**
+   * Remove shared object from cache
+   *
+   * @param <E> The type of the shared object
+   * @param name Name of the shared object to be removed
+   * @return The object that was removed
+   */
   <E> E removeSharedObject(String name);
 
   /**
@@ -66,12 +92,15 @@ public interface JobContext {
   void stopStreamingCtx();
 
   /**
-   * Returns a local tmp dir specific to the context
+   * @return A local tmp dir specific to the context
    */
   File getLocalTmpDir();
 
   /**
-   * Returns SparkSession if it existed, otherwise throws Exception.
+   *
+   * @param <E> The type of the sparksession object
+   * @return The SparkSession if it exists
+   * @throws Exception If SparkSession does not exist
    */
   <E> E sparkSession() throws Exception;
 }

api/src/main/java/org/apache/livy/JobHandle.java

@@ -27,6 +27,8 @@ public interface JobHandle<T> extends Future<T> {
 
   /**
    * Return the current state of the job.
+   *
+   * @return The current State of this job
    */
   State getState();
 
@@ -60,6 +62,8 @@ public interface JobHandle<T> extends Future<T> {
      * Notifies when a job has been queued for execution on the remote context. Note that it is
      * possible for jobs to bypass this state and got directly from the SENT state to the STARTED
      * state.
+     *
+     * @param job The JobHandle for the queued job
      */
     void onJobQueued(JobHandle<T> job);
 

api/src/main/java/org/apache/livy/LivyClient.java

@@ -29,6 +29,7 @@ public interface LivyClient {
   /**
    * Submits a job for asynchronous execution.
    *
+   * @param <T> The return type of the job
    * @param job The job to execute.
    * @return A handle that be used to monitor the job.
    */
@@ -36,16 +37,16 @@ public interface LivyClient {
 
   /**
    * Asks the remote context to run a job immediately.
-   * <p/>
+   * <p>
    * Normally, the remote context will queue jobs and execute them based on how many worker
    * threads have been configured. This method will run the submitted job in the same thread
    * processing the RPC message, so that queueing does not apply.
-   * <p/>
+   * <p>
    * It's recommended that this method only be used to run code that finishes quickly. This
    * avoids interfering with the normal operation of the context.
-   * <p/>
-   * Note: the {@link JobContext#monitor()} functionality is not available when using this method.
+   * <p>
    *
+   * @param <T> The return type of the job
    * @param job The job to execute.
    * @return A future to monitor the result of the job.
    */

api/src/main/java/org/apache/livy/LivyClientBuilder.java

@@ -40,6 +40,8 @@ public final class LivyClientBuilder {
   /**
    * Creates a new builder that will automatically load the default Livy and Spark configuration
    * from the classpath.
+   *
+   * @throws IOException If an error occurred when reading from the config files.
    */
   public LivyClientBuilder() throws IOException {
     this(true);
@@ -54,6 +56,10 @@ public final class LivyClientBuilder {
    * application's classpath. Livy configuration takes precedence over Spark's (in case
    * configuration entries are duplicated), and configuration set in this builder object will
    * override the values in those files.
+   *
+   * @param loadDefaults Whether to load configs from spark-defaults.conf and livy-client.conf
+   *                     if they are found in the application's classpath.
+   * @throws IOException If an error occurred when reading from the config files.
    */
   public LivyClientBuilder(boolean loadDefaults) throws IOException {
     this.config = new Properties();

api/src/main/java/org/apache/livy/LivyClientFactory.java

@@ -38,6 +38,8 @@ public interface LivyClientFactory {
    * Instantiates a new client if the given URI is supported by the implementation.
    *
    * @param uri URI pointing at the livy backend to use.
+   * @param config Livy client configs.
+   * @return The newly created LivyClient or null if an unsupported URI
    */
   LivyClient createClient(URI uri, Properties config);
 

api/src/main/java/org/apache/livy/package-info.java

@@ -0,0 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Livy programmatic Java API
+ */
+package org.apache.livy;

client-common/src/main/java/org/apache/livy/client/common/AbstractJobHandle.java

@@ -52,10 +52,13 @@ public abstract class AbstractJobHandle<T> implements JobHandle<T> {
    * Changes the state of this job handle, making sure that illegal state transitions are ignored.
    * Fires events appropriately.
    *
-   * As a rule, state transitions can only occur if the current state is "higher" than the current
+   * As a rule, state transitions can only occur if the new state is "higher" than the current
    * state (i.e., has a higher ordinal number) and is not a "final" state. "Final" states are
    * CANCELLED, FAILED and SUCCEEDED, defined here in the code as having an ordinal number higher
    * than the CANCELLED enum constant.
+   *
+   * @param newState The new state to change to.
+   * @return Whether the state changed.
    */
   public boolean changeState(State newState) {
     synchronized (listeners) {

client-common/src/main/java/org/apache/livy/client/common/ClientConf.java

@@ -40,12 +40,14 @@ public abstract class ClientConf<T extends ClientConf>
 
   public static interface ConfEntry {
 
-    /** The key in the configuration file. */
+    /**
+     * @return The key in the configuration file.
+     */
     String key();
 
     /**
-     * The default value, which also defines the type of the config. Supported types:
-     * Boolean, Integer, Long, String. <code>null</code> maps to String.
+     * @return The default value, which also defines the type of the config. Supported types:
+     *         Boolean, Integer, Long, String. <code>null</code> maps to String.
      */
     Object dflt();
 
@@ -226,10 +228,14 @@ public abstract class ClientConf<T extends ClientConf>
     }
   }
 
-  /** Maps valid key to DeprecatedConf with the deprecated key. */
+  /**
+   * @return A Map from a valid key to a DeprecatedConf with the deprecated key.
+   */
   protected abstract Map<String, DeprecatedConf> getConfigsWithAlternatives();
 
-  /** Maps deprecated key to DeprecatedConf with the same key. */
+  /**
+   * @return A Map from a deprecated key to a DeprecatedConf with the same key.
+   */
   protected abstract Map<String, DeprecatedConf> getDeprecatedConfigs();
 
   private static class ConfPair {
@@ -261,13 +267,19 @@ public abstract class ClientConf<T extends ClientConf>
 
   public static interface DeprecatedConf {
 
-    /** The key in the configuration file. */
+    /**
+     * @return The key in the configuration file.
+     */
     String key();
 
-    /** The Livy version in which the key was deprecated. */
+    /**
+     * @return The Livy version in which the key was deprecated.
+     */
     String version();
 
-    /** Message to include in the deprecation warning for configs without alternatives */
+    /**
+     * @return Message to include in the deprecation warning for configs without alternatives
+     */
     String deprecationMessage();
   }
 

docs/README.md

@@ -46,12 +46,27 @@ Before opening a pull request, you can preview your contributions by running fro
 3. Open http://localhost:4000
 ```
 
+## API Docs (Scaladocs and Javadocs)
+
+You can build just the Livy javadocs by running  `mvn javadoc:aggregate` from the `LIVY_HOME`
+directory, or the Livy scaladocs by running `mvn scala:doc` in certain modules. (Scaladocs do
+not currently build in every module)
+
+When you run `jekyll build` in the `docs` directory, it will also copy over the scaladocs and
+javadocs for the public APIs into the `docs` directory (and then also into the `_site` directory).
+We use a jekyll plugin to run the api builds before building the site so if you haven't run it
+(recently) it may take some time as it generates all of the scaladoc and javadoc using Maven.
+
+NOTE: To skip the step of building and copying over the Scala and Java API docs, run `SKIP_API=1
+jekyll build`.
+
+
 ## Publishing Docs to [livy.incubator.apache.org]
 
 1. Build Livy Docs (`cd docs` then `bundle exec jekyll build`).
 2. Copy the contents of `docs/target/` excluding `assets/` into a new directory (eg. `0.4.0/`) and
 move it into the `docs/` directory in your local fork of `apache/incubator-livy-website`.
-3. If this is a major version, update the `latest` symlink to point to the new directory.
+3. If nesesary, update the `latest` symlink to point to the new directory.
 4. Open a pull request to `apache/incubator-livy-website` with the update.
 
 Note: If you made any changes to files in the `assets/` directory you will need to replicate those

docs/_data/navigation.yml

@@ -21,6 +21,10 @@ topnav:
       url: rest-api.html
     - title: Programmatic API
       url: programmatic-api.html
+    - title: Java API Docs
+      url: api/java/index.html
+    - title: Scala API Docs
+      url: api/scala/index.html#org.apache.livy.scalaapi.package
 
 - title: Apache
   subcategories:

docs/_plugins/build-apis.rb

@@ -0,0 +1,62 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'fileutils'
+include FileUtils
+
+if not (ENV['SKIP_API'] == '1')
+  # Build Scaladoc for Scala API and Javadoc for Java API
+
+  puts "Moving to scala-api module and building Scala API docs."
+  cd("../scala-api")
+
+  puts "Running 'mvn scala:doc' from " + pwd + "; this may take a few minutes..."
+  system("mvn scala:doc") || raise("Scaladoc maven build failed")
+
+  puts "Moving to api module and building Java API docs."
+  cd("../api")
+
+  puts "Running 'mvn javadoc:javadoc -Ppublic-docs' from " + pwd + "; this may take a few minutes..."
+  system("mvn javadoc:javadoc -Ppublic-docs") || raise("Javadoc maven build failed")
+
+  puts "Moving back into docs dir."
+  cd("../docs")
+
+  puts "Removing old docs"
+  puts `rm -rf api`
+
+  # Copy over the ScalaDoc for the Scala API to api/scala.
+  # This directory will be copied over to _site when `jekyll` command is run.
+  source = "../scala-api/target/site/scaladocs"
+  dest = "api/scala"
+
+  puts "Making directory " + dest
+  mkdir_p dest
+
+  # From the rubydoc: cp_r('src', 'dest') makes src/dest, but this doesn't.
+  puts "cp -r " + source + "/. " + dest
+  cp_r(source + "/.", dest)
+
+  # Copy over the JavaDoc for the Java API to api/java.
+  source = "../api/target/site/apidocs"
+  dest = "api/java"
+
+  puts "Making directory " + dest
+  mkdir_p dest
+
+  puts "cp -r " + source + "/. " + dest
+  cp_r(source + "/.", dest)
+
+end

pom.xml

@@ -792,6 +792,16 @@
           <version>1.13</version>
         </plugin>
 
+        <plugin>
+          <groupId>org.apache.maven.plugins</groupId>
+          <artifactId>maven-javadoc-plugin</artifactId>
+          <version>3.0.0-M1</version>
+          <configuration>
+            <doctitle>Livy ${project.version} Java API</doctitle>
+            <windowtitle>Livy ${project.version} JavaDoc</windowtitle>
+          </configuration>
+        </plugin>
+
         <plugin>
           <groupId>com.googlecode.maven-download-plugin</groupId>
           <artifactId>download-maven-plugin</artifactId>
@@ -1205,6 +1215,27 @@
     </build>
   </profile>
 
+    <profile>
+      <id>public-docs</id>
+      <activation>
+        <property>
+          <name>public-docs</name>
+        </property>
+      </activation>
+      <build>
+        <plugins>
+          <plugin>
+            <groupId>org.apache.maven.plugins</groupId>
+            <artifactId>maven-javadoc-plugin</artifactId>
+            <configuration>
+              <failOnError>true</failOnError>
+              <excludePackageNames>*.test,*.example,*.client,*.annotations,*.rsc</excludePackageNames>
+            </configuration>
+          </plugin>
+        </plugins>
+      </build>
+    </profile>
+
   </profiles>
 
 </project>

rsc/src/main/java/org/apache/livy/rsc/driver/RSCDriver.java

@@ -273,6 +273,8 @@ public class RSCDriver extends BaseProtocol {
    * context with the provided configuration. Subclasses can override this behavior,
    * and returning a null context is allowed. In that case, the context exposed by
    * JobContext will be null.
+   *
+   * @return The initalized SparkContext
    */
   protected SparkEntries initializeSparkEntries() throws Exception {
     SparkEntries entries = new SparkEntries(conf);

rsc/src/main/java/org/apache/livy/rsc/package-info.java

@@ -0,0 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Livy Remote Spark Context used by Livy's Interactive Sessions
+ */
+package org.apache.livy.rsc;

rsc/src/main/java/org/apache/livy/rsc/rpc/Rpc.java

@@ -241,6 +241,9 @@ public class Rpc implements Closeable {
   /**
    * Send an RPC call to the remote endpoint and returns a future that can be used to monitor the
    * operation.
+   *
+   * @param msg RPC call to send.
+   * @return A future used to monitor the operation.
    */
   public Future<Void> call(Object msg) {
     return call(msg, Void.class);

rsc/src/main/java/org/apache/livy/rsc/rpc/RpcDispatcher.java

@@ -36,11 +36,11 @@ import org.apache.livy.rsc.Utils;
 /**
  * An implementation of ChannelInboundHandler that dispatches incoming messages to an instance
  * method based on the method signature.
- * <p/>
+ * <p>
  * A handler's signature must be of the form:
- * <p/>
+ * <p>
  * <blockquote><tt>protected void handle(ChannelHandlerContext, MessageType)</tt></blockquote>
- * <p/>
+ * <p>
  * Where "MessageType" must match exactly the type of the message to handle. Polymorphism is not
  * supported. Handlers can return a value, which becomes the RPC reply; if a null is returned, then
  * a reply is still sent, with an empty payload.
@@ -54,7 +54,10 @@ public abstract class RpcDispatcher extends SimpleChannelInboundHandler<Object>
 
   private volatile Rpc.MessageHeader lastHeader;
 
-  /** Override this to add a name to the dispatcher, for debugging purposes. */
+  /**
+   * Override this to add a name to the dispatcher, for debugging purposes.
+   * @return The name of this dispatcher.
+   */
   protected String name() {
     return getClass().getSimpleName();
   }

rsc/src/main/java/org/apache/livy/rsc/rpc/RpcServer.java

@@ -74,7 +74,7 @@ public class RpcServer implements Closeable {
   private final String PORT_DELIMITER = "~";
   /**
    * Creating RPC Server
-   * @param lconf
+   * @param lconf The default RSC configs
    * @throws IOException
    * @throws InterruptedException
    */
@@ -192,6 +192,8 @@ public class RpcServer implements Closeable {
 
   /**
    * Creates a secret for identifying a client connection.
+   *
+   * @return the secret
    */
   public String createSecret() {
     byte[] secret = new byte[config.getInt(RPC_SECRET_RANDOM_BITS) / 8];