Merge pull request #3140 from blink1073/pip-install

Build Cleanup

.travis.yml

@@ -11,8 +11,8 @@ cache:
 env:
   matrix:
     - GROUP=tests
-    - GROUP=coverage_and_docs
-    - GROUP=cli
+    - GROUP=coverage
+    - GROUP=other
   global:
   - GH_REF: github.com/jupyterlab/jupyterlab.git
   - secure: MWpTI6cj3/Bnmtrr0Oqlp2JeWqDneB9aEjlQDaRxLOkqVbxhqDcYW9qAgZZP+sq29vT5oVMWzyCirteKxJfG2vy3HQE1XNLhz82Sf/7sE6DQ51gohl0CcOeA/uA8hCXEw97hneFWsZgHKqSoch7nVDsE3qfYgO+930jHlnxYApJGP9hZFv2Q2NVa6+99kipEYS4BY/yBDYKy6/t4kXcnBrUlNaPtdjnXcrY9esLZ7EQtkaG5VqcQVIBaLJKGF5Q7Aufj5nCFaZ6hZDF1Bi/AbmIbVWFyiT+22i8DZK6YwenECckyzoWkl+bEhYepWsgBKh/BDgPBAmPWKHgU5V4apDaGqZBhF7FP6H02AdZYYuCwl47jyakqvWLZW7oDmorL+HsWG5HQ3m0tMT2ywdbwNOiD39tiPPXjsvROh5ys9vL6NzQvxILCeEOnzcZrFuxi2LGEZfnlqRIjkh1llUAvNc3mOycRLWDOwVQa2+U59qDRXCSY2RD+MOfcdFUGengVujTMaAPMBUa3E33/ZIOOKJtR5TIajYZvd9B2uDlz02QfvTK+hrTaNYJjRZ8WCaeSM/CIKdoLw+29MNO6eqtchw0/vNvM8c9EkhrhMQKcY04OecVhmZkemFhd4SD5l92VX3z3xSxLkmazfNkj3CigWDXNxfDd2ORoGjA46Pga8RM=

CONTRIBUTING.md

@@ -73,18 +73,17 @@ steps:
 ```bash
 git clone https://github.com/<your-github-username>/jupyterlab.git
 cd jupyterlab
-pip install -e . # will take a long time to build everything
+pip install -e .
 npm install
-npm run build:main
+npm run build  # Build the dev mode assets
+jupyter lab build  # Build the app dir assets
 jupyter serverextension enable --py jupyterlab
 ```
 
 Notes:
 
-* At times, it may be necessary to clean your local repo with the command ``git
-clean -fdx``.
-
-* Make sure to do `pip install -e` *before* running `npm install`. Running `npm install` first makes pip stall.
+* At times, it may be necessary to clean your local repo with the command `npm run clean:slate`.  This will clean the repository, and re-install and 
+rebuild.  If using a local install, you will need to re-run `pip install -e .`
 
 * If `pip` gives a `VersionConflict` error, it usually means that the installed
 version of `jupyterlab_launcher` is out of date. Run `pip install --upgrade

README.md

@@ -54,11 +54,17 @@ Note: If installing using `pip install --user`, you must add the user-level
  `bin` directory to your `PATH` environment variable in order to launch
  `jupyter lab`.
 
-Note: If installing from source and not using the
-development installation (`pip install -e .`), you
-must run `jupyter lab build` after the install to
-get a working application directory.
+Note: JupyterLab can be installed from a git checkout using `pip`.  Note
+that you will have to build the application after installing to get the static 
+assets.  `pip` will warn you that the static assets are not included when 
+installing, and it will fail to build a cached wheel if it decides to try 
+and make one.  Example:
 
+```bash
+pip install git+git://github.com/jupyterlab/jupyterlab.git
+jupyter lab build   # Requires nodejs
+jupyter serverextension enable --py jupyterlab --sys-prefix
+```
 
 ### Running
 

RELEASE.md

@@ -8,18 +8,6 @@ This document guides a contributor through creating a release of JupyterLab.
 Review ``CONTRIBUTING.md``. Make sure all the tools needed to generate the
 built JavaScript files are properly installed.
 
-## Clean the repository
-
-You can remove all non-tracked files with:
-
-```bash
-git clean -xfdi
-```
-
-This would ask you for confirmation before removing all untracked files. Make
-sure the ``dist/`` folder is clean and avoid stale build from
-previous attempts.
-
 ## Create the release
 
 We publish the npm packages, a Python source package, and a Python universal binary wheel.  We also publish a conda package on conda-forge (see below).
@@ -43,7 +31,7 @@ npm run publish
 - Prep the static assets for release:
 
 ```bash
-cd jupyterlab && npm run publish && cd ..
+npm run build:static
 ```
 
 - Commit and tag and push the tag

jupyterlab/commands.py

@@ -49,20 +49,18 @@ def get_app_dir(app_dir=None):
         return os.path.realpath(app_dir)
 
     # Use the default locations for data_files.
+    app_dir = pjoin(sys.prefix, 'share', 'jupyter', 'lab')
 
     # Check for a user level install.
     if hasattr(site, 'getuserbase') and here.startswith(site.getuserbase()):
         app_dir = pjoin(site.getuserbase(), 'share', 'jupyter', 'lab')
 
     # Check for a system install in '/usr/local/share'.
-    elif (sys.prefix == '/usr' and
+    elif (sys.prefix.startswith('/usr') and not
+          os.path.exists(app_dir) and
           os.path.exists('/usr/local/share/jupyter/lab')):
         app_dir = '/usr/local/share/jupyter/lab'
 
-    # Otherwise use the sys prefix.
-    else:
-        app_dir = pjoin(sys.prefix, 'share', 'jupyter', 'lab')
-
     return os.path.realpath(app_dir)
 
 

jupyterlab/package.json

@@ -5,8 +5,8 @@
   "scripts": {
     "build": "npm run clean && node update-core.js && webpack",
     "build:prod": "node update-core.js && webpack --devtool source-map",
+    "build:static": "node make-release.js",
     "clean": "rimraf build",
-    "publish": "node make-release.js",
     "watch": "node update-core.js && webpack --watch"
   },
   "dependencies": {

package.json

@@ -8,6 +8,7 @@
     "build:main": "npm run build",
     "build:main:prod": "npm run build:packages && cd jupyterlab && npm run build:prod",
     "build:src": "lerna run build --scope \"@jupyterlab/!(test-|example-)*\"",
+    "build:static": "cd jupyterlab && npm run build:static",
     "build:test": "lerna run build:test",
     "clean": "node scripts/clean-packages.js examples packages",
     "clean:examples": "node scripts/clean-packages.js examples",
@@ -19,7 +20,7 @@
     "docs": "lerna run docs",
     "get:dependency": "node scripts/get-dependency.js",
     "integrity": "node scripts/package-integrity.js",
-    "publish": "npm update && npm install && npm run clean && npm run build:packages && lerna publish --force-publish=* -m \"Publish\"",
+    "publish": "npm run clean:slate && lerna publish --force-publish=* -m \"Publish\"",
     "test": "cd test && npm test",
     "test:services": "cd packages/services && npm test && npm run test:integration && cd examples/node && python main.py",
     "test:chrome": "lerna run test:chrome --stream",

scripts/travis_install.sh

@@ -6,11 +6,11 @@ set -x
 
 npm update
 
-if [[ $GROUP == tests || $GROUP == cli ]]; then
+if [[ $GROUP == tests || $GROUP == other ]]; then
     wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh;
 fi
 
-if [[ $GROUP == coverage_and_docs ]]; then
+if [[ $GROUP == coverage ]]; then
     wget https://repo.continuum.io/miniconda/Miniconda2-latest-Linux-x86_64.sh -O miniconda.sh;
 fi
 
@@ -32,6 +32,5 @@ pip install -v -e ".[test]"
 test -e jupyterlab/schemas/jupyter.extensions.shortcuts.json
 test -e jupyterlab/themes/jupyterlab-theme-light-extension/images/jupyterlab.svg
 npm install
-npm run build:main
+npm run build
 jupyter serverextension enable --py jupyterlab
-jupyter lab build

scripts/travis_script.sh

@@ -10,71 +10,41 @@ sh -e /etc/init.d/xvfb start || true
 export PATH="$HOME/miniconda/bin:$PATH"
 
 
-npm run integrity
-npm run build:examples
-
-
 if [[ $GROUP == tests ]]; then
 
     # Run the JS and python tests
     py.test
-    npm run clean
-    npm run build:src
     npm run build:test
     npm test
     npm run test:services || npm run test:services
 
-    # Make sure we have CSS that can be converted with postcss
-    npm install -g postcss-cli
-    postcss packages/**/style/*.css --dir /tmp
-
-    # Run the publish script in jupyterlab
-    cd jupyterlab
-    npm run publish
-
-    if [ ! -f ./build/release_data.json ]; then
-        echo "npm publish in jupyterlab unsucessful!"
-    fi
 fi
 
 
-if [[ $GROUP == coverage_and_docs ]]; then
+if [[ $GROUP == coverage ]]; then
+
     # Run the coverage and python tests.
     py.test
-    npm run build
     npm run build:test
     npm run coverage
+    npm run clean
 
-    # Run the link check
-    pip install -q pytest-check-links
-    py.test --check-links -k .md .
-
-    # Build the api docs
-    npm run docs
-    cp jupyter_plugins.png docs
-
-    # Verify tutorial docs build
-    pushd docs
-    conda env create -n test_docs -f environment.yml
-    source activate test_docs
-    make html
-    source deactivate
-    popd
 fi
 
 
-if [[ $GROUP == cli ]]; then
+if [[ $GROUP == other ]]; then
+
     # Make sure we can successfully load the core app.
     pip install selenium
     python -m jupyterlab.selenium_check --core-mode
 
-    # Make sure we can build and run the app.
+    # Make sure we can run the built app.
     jupyter lab build
     python -m jupyterlab.selenium_check 
     jupyter labextension list
 
     # Make sure we can non-dev install.
-    conda create -n test_install notebook
+    conda create -n test_install notebook python=3.5
     source activate test_install
     pip install .
     jupyter lab build
@@ -110,4 +80,38 @@ if [[ $GROUP == cli ]]; then
     jupyter labextension list -h
     jupyter labextension enable -h
     jupyter labextension disable -h
+
+    # Run the package integrity check
+    npm run integrity
+
+    # Make sure the examples build
+    npm run build:examples
+
+    # Run the link check
+    pip install -q pytest-check-links
+    py.test --check-links -k .md .
+
+    # Build the api docs
+    npm run docs
+    cp jupyter_plugins.png docs
+
+    # Verify tutorial docs build
+    pushd docs
+    conda env create -n test_docs -f environment.yml
+    source activate test_docs
+    make html
+    source deactivate
+    popd
+
+    # Make sure we have CSS that can be converted with postcss
+    npm install -g postcss-cli
+    postcss packages/**/style/*.css --dir /tmp
+
+    # Make sure we can make release assets
+    npm run build:static
+    if [ ! -f ./build/release_data.json ]; then
+        echo "npm publish in jupyterlab unsucessful!"
+    fi
+    python setup.py sdist
+    python setup.py bdist_wheel --universal
 fi

setupbase.py

@@ -12,10 +12,10 @@ This includes:
 import io
 import json
 import os
-import glob
 import pipes
 import sys
 import shutil
+import tempfile
 import os.path as osp
 from os.path import join as pjoin
 
@@ -210,14 +210,18 @@ class bdist_egg_disabled(bdist_egg):
 
 
 class custom_egg_info(egg_info):
-    """Prune node_modules folders from egg_info to avoid symlink recursion
+    """Prune JavaScript folders from egg_info to avoid locking up pip.
     """
+
     def run(self):
-        folders = []
-        for dir, subdirs, files in os.walk(here):
-            if 'node_modules' in subdirs:
-                folders.append(pjoin(dir, 'node_modules'))
-                subdirs.remove('node_modules')
+        folders = ['examples', 'packages', 'test', 'node_modules']
+        if not os.path.exists(pjoin(here, 'node_modules')):
+            folders.remove('node_modules')
+        tempdir = tempfile.mkdtemp()
+        for folder in folders:
+            shutil.move(pjoin(here, folder), tempdir)
+        value = egg_info.run(self)
         for folder in folders:
-            shutil.rmtree(folder)
-        return egg_info.run(self)
+            shutil.move(pjoin(tempdir, folder), here)
+        shutil.rmtree(tempdir)
+        return value