Merge pull request #2808 from blink1073/ensure-extra-files

Clean up handling of schema and theme files

jupyterlab/make-release.js

@@ -10,15 +10,8 @@ var cwd = path.resolve('..');
 var version = childProcess.execSync('python setup.py --version', { cwd: cwd });
 version = version.toString().trim();
 
-// Update our package.json files.
+// Update the package.app.json file.
 var data = require('./package.json');
-data['jupyterlab']['version'] = version;
-
-// Update our package.json files.
-var text = JSON.stringify(sortPackageJson(data), null, 2) + '\n';
-fs.writeFileSync('./package.json', text);
-
-// Update the build script.
 data['scripts']['build'] = 'webpack'
 text = JSON.stringify(sortPackageJson(data), null, 2) + '\n';
 fs.writeFileSync('./package.app.json', text);

scripts/travis_install.sh

@@ -28,6 +28,9 @@ mkdir ~/.jupyter
 
 # Install and enable the server extension
 pip install -v -e ".[test]"
+# Make sure the schema and theme files exist
+test -e jupyterlab/schemas/jupyter.extensions.shortcuts.json
+test -e jupyterlab/themes/jupyterlab-theme-light-extension/images/jupyterlab.svg
 npm install
 npm run build:main
 jupyter serverextension enable --py jupyterlab

setup.py

@@ -28,28 +28,31 @@ from distutils import log
 import json
 import os
 from glob import glob
+
+# BEFORE importing distutils, remove MANIFEST. distutils doesn't properly
+# update it when the contents of directories change.
+if os.path.exists('MANIFEST'): os.remove('MANIFEST')
+
 from distutils.command.build_ext import build_ext
+from distutils.command.build_py import build_py
 from setuptools.command.sdist import sdist
 from setuptools import setup
 from setuptools.command.bdist_egg import bdist_egg
 
 
 # Our own imports
-
 from setupbase import (
     bdist_egg_disabled,
+    ensure_core_data,
     find_packages,
     find_package_data,
     js_prerelease,
     CheckAssets,
+    CoreDeps,
     version_ns,
     name
 )
 
-# BEFORE importing distutils, remove MANIFEST. distutils doesn't properly
-# update it when the contents of directories change.
-if os.path.exists('MANIFEST'): os.remove('MANIFEST')
-
 
 here = os.path.dirname(os.path.abspath(__file__))
 pjoin = os.path.join
@@ -89,10 +92,12 @@ setup_args = dict(
 
 
 cmdclass = dict(
-    build_ext = build_ext,
+    build_py = ensure_core_data(build_py),
+    build_ext = ensure_core_data(build_ext),
     sdist  = js_prerelease(sdist, strict=True),
     bdist_egg = bdist_egg if 'bdist_egg' in sys.argv else bdist_egg_disabled,
-    jsdeps = CheckAssets
+    jsdeps = CheckAssets,
+    coredeps = CoreDeps,
 )
 try:
     from wheel.bdist_wheel import bdist_wheel

setupbase.py

@@ -12,8 +12,11 @@ This includes:
 import io
 import json
 import os
+import glob
 import pipes
 import sys
+import shutil
+import os.path as osp
 from os.path import join as pjoin
 
 from distutils import log
@@ -33,8 +36,8 @@ else:
 name = 'jupyterlab'
 
 
-here = os.path.dirname(os.path.abspath(__file__))
-is_repo = os.path.exists(pjoin(here, '.git'))
+here = osp.dirname(osp.abspath(__file__))
+is_repo = osp.exists(pjoin(here, '.git'))
 
 version_ns = {}
 with io.open(pjoin(here, name, '_version.py'), encoding="utf8") as f:
@@ -58,7 +61,7 @@ def find_packages():
     """
     packages = []
     for dir, subdirs, files in os.walk('jupyterlab'):
-        package = dir.replace(os.path.sep, '.')
+        package = dir.replace(osp.sep, '.')
         if '__init__.py' not in files:
             # not a package
             continue
@@ -81,13 +84,40 @@ def find_package_data():
     }
 
 
+def ensure_core_data(command):
+    """decorator for building minified js/css prior to another command"""
+    class DecoratedCommand(command):
+
+        def run(self):
+            coredeps = self.distribution.get_command_obj('coredeps')
+            if not is_repo and all(osp.exists(t) for t in coredeps.targets):
+                # build_py or build_ext, nothing to do
+                command.run(self)
+                return
+
+            try:
+                self.distribution.run_command('coredeps')
+            except Exception as e:
+                missing = [t for t in coredeps.targets if not osp.exists(t)]
+                if missing:
+                    log.warn('file check failed')
+                    if missing:
+                        log.error('missing files: %s' % missing)
+                    raise e
+                else:
+                    log.warn('core deps check failed (not a problem)')
+                    log.warn(str(e))
+            command.run(self)
+    return DecoratedCommand
+
+
 def js_prerelease(command, strict=False):
     """decorator for building minified js/css prior to another command"""
     class DecoratedCommand(command):
 
         def run(self):
             jsdeps = self.distribution.get_command_obj('jsdeps')
-            if not is_repo and all(os.path.exists(t) for t in jsdeps.targets):
+            if not is_repo and all(osp.exists(t) for t in jsdeps.targets):
                 # sdist, nothing to do
                 command.run(self)
                 return
@@ -95,7 +125,7 @@ def js_prerelease(command, strict=False):
             try:
                 self.distribution.run_command('jsdeps')
             except Exception as e:
-                missing = [t for t in jsdeps.targets if not os.path.exists(t)]
+                missing = [t for t in jsdeps.targets if not osp.exists(t)]
                 if strict or missing:
                     log.warn('js check failed')
                     if missing:
@@ -114,6 +144,66 @@ def update_package_data(distribution):
     build_py.finalize_options()
 
 
+class CoreDeps(Command):
+    description = 'ensure required core assets'
+
+    user_options = []
+
+    # Representative files that should exist after a successful core setup
+    targets = [
+        pjoin(here, 'jupyterlab', 'schemas', 'jupyter.extensions.shortcuts.json'),
+        pjoin(here, 'jupyterlab', 'themes', 'jupyterlab-theme-light-extension',
+            'images', 'jupyterlab.svg')
+    ]
+
+    def initialize_options(self):
+        pass
+
+    def finalize_options(self):
+        pass
+
+    def run(self):
+        # Handle schemas.
+        schema = pjoin(here, 'jupyterlab', 'schemas')
+        if osp.exists(schema):
+            shutil.rmtree(schema)
+        os.makedirs(schema)
+
+        packages = glob.glob(pjoin(here, 'packages', '**', 'package.json'))
+        for pkg in packages:
+            with open(pkg) as fid:
+                data = json.load(fid)
+            if 'jupyterlab' not in data:
+                continue
+            schemaDir = data['jupyterlab'].get('schemaDir', '')
+            if schemaDir:
+                parentDir = osp.dirname(pkg)
+                files = glob.glob(pjoin(parentDir, schemaDir, '*.json'))
+                for file in files:
+                    shutil.copy(file, schema)
+
+        # Handle themes
+        themes = pjoin(here, 'jupyterlab', 'themes')
+        if osp.exists(themes):
+            shutil.rmtree(themes)
+        os.makedirs(themes)
+
+        packages = glob.glob(pjoin(here, 'packages', '**', 'package.json'))
+        for pkg in packages:
+            with open(pkg) as fid:
+                data = json.load(fid)
+            if 'jupyterlab' not in data:
+                continue
+            themeDir = data['jupyterlab'].get('themeDir', '')
+            if themeDir:
+                name = data['name'].replace('@', '').replace('/', '-')
+                src = pjoin(osp.dirname(pkg), themeDir)
+                shutil.copytree(src, pjoin(themes, name))
+
+        # update package data in case this created new files
+        update_package_data(self.distribution)
+
+
 class CheckAssets(Command):
     description = 'check for required assets'
 
@@ -123,7 +213,7 @@ class CheckAssets(Command):
     targets = [
         pjoin(here, 'jupyterlab', 'build', 'release_data.json'),
         pjoin(here, 'jupyterlab', 'build', 'main.bundle.js'),
-    ]
+    ] + CoreDeps.targets
 
     def initialize_options(self):
         pass
@@ -133,7 +223,7 @@ class CheckAssets(Command):
 
     def run(self):
         for t in self.targets:
-            if not os.path.exists(t):
+            if not osp.exists(t):
                 msg = 'Missing file: %s' % t
                 raise ValueError(msg)
 
@@ -146,7 +236,6 @@ class CheckAssets(Command):
             msg = 'Release assets version mismatch, please run npm publish'
             raise ValueError(msg)
 
-
         # update package data in case this created new files
         update_package_data(self.distribution)