Add Ability Use Source Directories in App Dir (#10024)

* Add Ability Use Source Directories in App Dir

* Update binder/jupyter_notebook_config.py

Co-authored-by: Jeremy Tuloup <jeremy.tuloup@gmail.com>

Co-authored-by: Steven Silvester <steven.silvester@ieee.org>
Co-authored-by: Jeremy Tuloup <jeremy.tuloup@gmail.com>

.github/workflows/linuxtests.yml

@@ -7,7 +7,7 @@ jobs:
     name: Linux
     strategy:
       matrix:
-        group: [integrity, integrity2, integrity3, release_check, docs, usage, usage2, python, examples, interop, nonode, linkcheck, lint]
+        group: [integrity, integrity2, integrity3, release_check, docs, usage, usage2, splice_source, python, examples, interop, nonode, linkcheck, lint]
         python: [3.6, 3.8]
         include:
           - group: release_check
@@ -35,6 +35,8 @@ jobs:
             python: 3.6
           - group: examples
             python: 3.6
+          - group: splice_source
+            python: 3.6
       fail-fast: false
     timeout-minutes: 30
     runs-on: ubuntu-20.04

binder/jupyter_notebook_config.py

@@ -1,28 +1,53 @@
-lab_command = ' '.join([
-    'jupyter',
-    'lab',
-    '--dev-mode',
-    '--debug',
-    '--extensions-in-dev-mode',
+common = [
     '--no-browser',
+    '--debug',
     '--port={port}',
     '--ServerApp.ip=127.0.0.1',
     '--ServerApp.token=""',
-    '--ServerApp.base_url={base_url}lab-dev',
     # Disable dns rebinding protection here, since our 'Host' header
     # is not going to be localhost when coming from hub.mybinder.org
     '--ServerApp.allow_remote_access=True'
-])
+]
+
+lab_command = ' '.join([
+    'jupyter',
+    'lab',
+    '--dev-mode',
+    '--extensions-in-dev-mode',
+    '--ServerApp.base_url={base_url}lab-dev',
+] + common + ['>jupyterlab-dev.log 2>&1'])
+
+
+lab_splice_command = ' '.join([
+    'jupyter',
+    'lab',
+    'build',
+    '--splice-source',
+    '--minimize=False',
+    '--dev-build=True',
+    '--debug',
+    '>jupyterlab-spliced.log 2>&1',
+    '&&',
+    'jupyter',
+    'lab',
+    '--ServerApp.base_url={base_url}lab-spliced',
+] + common + ['>jupyterlab-spliced.log 2>&1'])
+
 
 c.ServerProxy.servers = {
     'lab-dev': {
         'command': [
-            '/bin/bash', '-c',
-            # Redirect all logs to a log file
-            f'{lab_command} >jupyterlab-dev.log 2>&1'
+            '/bin/bash', '-c', lab_command
         ],
         'timeout': 60,
         'absolute_url': True
+    },
+    'lab-spliced': {
+        'command': [
+            '/bin/bash', '-c', lab_splice_command
+        ],
+        'timeout': 300,
+        'absolute_url': True
     }
 }
 

binder/postBuild

@@ -5,4 +5,4 @@ pip install -e .
 
 jlpm
 
-jlpm build
+jlpm build

dev_mode/webpack.config.js

@@ -1,3 +1,4 @@
+// This file is auto-generated from the corresponding file in /dev_mode
 /* -----------------------------------------------------------------------------
 | Copyright (c) Jupyter Development Team.
 | Distributed under the terms of the Modified BSD License.
@@ -86,7 +87,12 @@ Object.keys(jlab.linkedPackages).forEach(function (name) {
   if (name in watched) {
     return;
   }
-  const localPkgPath = require.resolve(path.join(name, 'package.json'));
+  let localPkgPath = '';
+  try {
+    localPkgPath = require.resolve(path.join(name, 'package.json'));
+  } catch (e) {
+    return;
+  }
   watched[name] = path.dirname(localPkgPath);
   if (localPkgPath.indexOf('node_modules') !== -1) {
     watchNodeModules = true;
@@ -105,7 +111,13 @@ const sourceMapRes = Object.values(watched).reduce((res, name) => {
  * and has no effect in `jupyter lab --dev-mode --watch`.
  */
 function maybeSync(localPath, name, rest) {
-  const stats = fs.statSync(localPath);
+  let stats;
+  try {
+    stats = fs.statSync(localPath);
+  } catch (e) {
+    return;
+  }
+
   if (!stats.isFile(localPath)) {
     return;
   }

docs/source/developer/repo.rst

@@ -53,6 +53,10 @@ the repository to be tested on `mybinder.org <https://mybinder.org>`__.
 This specification is developer focused.
 For a more user-focused binder see the
 `JupyterLab demo <https://mybinder.org/v2/gh/jupyterlab/jupyterlab-demo/master?urlpath=lab/tree/demo/Lorenz.ipynb>`__
+The binder instance adds two endpoints in addition to ``/lab``: ``/lab-dev`` and ``/lab-spliced``.
+The ``lab-dev`` endpoint is the equivalent of checking out the repo locally and running ``jupyter lab --dev-mode``.
+The ``lab-spliced`` endpoint is the equivalent of building JupyterLab in spliced mode and running ``jupyter lab``.
+See the `Development workflow for source extensions <../extension/extension_dev.html#development-workflow-for-source-extensions>`__ for more information on spliced mode.
 
 Build utilities: ``builtutils/``
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
@@ -122,5 +126,5 @@ A small ``npm`` package which is aids in running the tests in ``tests/``.
 TypeDoc Theming: ``typedoc-theme``
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-A small theme used to help render our 
+A small theme used to help render our
 `TypeDoc <../api/index.html>`__ documentation.

docs/source/extension/extension_dev.rst

@@ -139,7 +139,7 @@ Since consumers will need to import a token used by a provider, the token should
 A pattern in core JupyterLab is to create and export a token from a third package that both the provider and consumer extensions import, rather than defining the token in the provider's package. This enables a user to swap out the provider extension for a different extension that provides the same token with an alternative service implementation. For example, the core JupyterLab ``filebrowser`` package exports a token representing the file browser service (enabling interactions with the file browser). The ``filebrowser-extension`` package contains a plugin that implements the file browser in JupyterLab and provides the file browser service to JupyterLab (identified with the token imported from the ``filebrowser`` package). Extensions in JupyterLab that want to interact with the filebrowser thus do not need to have a JavaScript dependency on the ``filebrowser-extension`` package, but only need to import the token from the ``filebrowser`` package. This pattern enables users to seamlessly change the file browser in JupyterLab by writing their own extension that imports the same token from the ``filebrowser`` package and provides it to the system with their own alternative file browser service.
 
 
-.. 
+..
    We comment out the following, until we can import from a submodule of a package. See https://github.com/jupyterlab/jupyterlab/pull/9475.
 
    A pattern in core JupyterLab is to create and export tokens from a self-contained ``tokens`` JavaScript module in a package. This enables consumers to import a token directly from the package's ``tokens`` module (e.g., ``import { MyToken } from 'provider/tokens';``), thus enabling a tree-shaking bundling optimization to possibly bundle only the tokens and not other code from the package.
@@ -153,7 +153,7 @@ Mime Renderer Plugins
 
 Mime Renderer plugins are a convenience for creating a plugin
 that can render mime data in a notebook and files of the given mime type. Mime renderer plugins are more declarative and more restricted than standard plugins.
-A mime renderer plugin is an object with the fields listed in the 
+A mime renderer plugin is an object with the fields listed in the
 `rendermime-interfaces IExtension <../api/interfaces/rendermime_interfaces.irendermime.iextension.html>`__
 object.
 
@@ -172,7 +172,7 @@ document can then be saved by the user in the usual manner.
 Theme plugins
 ^^^^^^^^^^^^^
 
-A theme is a special application plugin that registers a theme with the ``ThemeManager`` service. Theme CSS assets are specially bundled in an extension (see :ref:`themePath`) so they can be unloaded or loaded as the theme is activated. 
+A theme is a special application plugin that registers a theme with the ``ThemeManager`` service. Theme CSS assets are specially bundled in an extension (see :ref:`themePath`) so they can be unloaded or loaded as the theme is activated.
 
 The extension package containing the theme plugin must include all static assets that are referenced by ``@import`` in its theme CSS files. Local URLs can be used to reference files relative to the location of the referring sibling CSS files. For example ``url('images/foo.png')`` or ``url('../foo/bar.css')`` can be used to refer local files in the theme. Absolute URLs (starting with a ``/``) or external URLs (e.g. ``https:``) can be used to refer to external assets.
 
@@ -205,7 +205,7 @@ We will talk about each ``jupyterlab`` metadata field in ``package.json`` for so
 * ``sharedPackages``: :ref:`deduplication`
 * ``discovery``: :ref:`ext-author-companion-packages`
 
-A JupyterLab extension must have at least one of ``jupyterlab.extension`` or ``jupyterlab.mimeExtension`` set. 
+A JupyterLab extension must have at least one of ``jupyterlab.extension`` or ``jupyterlab.mimeExtension`` set.
 
 .. _main_entry_point:
 
@@ -353,12 +353,12 @@ When an extension (the "consumer") is optionally using a service identified by a
 .. TODO: fill out the following text to a more complete explanation of how the deduplication works.
 
    Prebuilt extensions need to deduplicate many of their dependencies with other prebuilt extensions and with source extensions. This deduplication happens in two phases:
-   
+
    1. When JupyterLab is initialized in the browser, the core Jupyterlab build (including all source extensions) and each prebuilt extension can share copies of dependencies with a package cache in the browser.
    2. A source or prebuilt extension can import a dependency from the cache while JupyterLab is running.
-   
+
    The main options controlling how things work in this deduplication are as follows. If a package is listed in this sharing config, it will be requested from the package cache.
-   
+
    * ``bundled`` - if true, a copy of this package is also provided to the package cache. If false, we will request a version from the package cache. Set this to false if we know that the package cache will have the package and you do not want to bundle a copy (perhaps to make your prebuilt bundle smaller).
    ``singleton`` - if true, makes sure to use the same copy of a dependency that others are using, even if it is not the right version.
    ``strictVersion`` - if true, throw an error if we would be using the wrong version of a dependency.
@@ -556,6 +556,8 @@ This ``install.json`` file is used by JupyterLab to help a user know how to mana
 
 
 
+.. _source_dev_workflow:
+
 Development workflow for source extensions
 ------------------------------------------
 
@@ -601,23 +603,14 @@ specific patch release of one of the core JupyterLab packages you can
 temporarily pin that requirement to a specific version in your own
 dependencies.
 
-If you must install a source extension into a development branch of JupyterLab, you have to graft it into the source tree of JupyterLab itself. This may be done using the command
-
-::
-
-    jlpm run add:sibling <path-or-url>
-
-in the JupyterLab root directory, where ``<path-or-url>`` refers either
-to an extension ``npm`` package on the local file system, or a URL to a git
-repository for an extension ``npm`` package. This operation may be
-subsequently reversed by running
+If you want to test a source extension against the unreleased versions of JupyterLab, you can run the command
 
 ::
 
-    jlpm run remove:package <extension-dir-name>
+    jupyter lab --watch --splice-source
 
-This will remove the package metadata from the source tree and delete
-all of the package files. Note that :ref:`developing a prebuilt extension <prebuilt_dev_workflow>` against a development version of JupyterLab is generally much easier.
+This command will splice the local ``packages`` directory into the application directory, allowing you to build source extension(s)
+against the current development sources.  To statically build spliced sources, use ``jupyter lab build --splice-source``.  Once a spliced build is created, any subsquent calls to `jupyter labextension build` will be in splice mode by default.  A spliced build can be forced by calling ``jupyter labextension build --splice-source``. Note that :ref:`developing a prebuilt extension <prebuilt_dev_workflow>` against a development version of JupyterLab is generally much easier than source package building.
 
 The package should export EMCAScript 6 compatible JavaScript. It can
 import CSS using the syntax ``require('foo.css')``. The CSS files can

jupyterlab/commands.py

@@ -39,12 +39,18 @@ from traitlets import Bool, Dict, HasTraits, Instance, List, Unicode, default
 from jupyterlab.coreconfig import CoreConfig
 from jupyterlab.jlpmapp import HERE, YARN_PATH
 from jupyterlab.semver import Range, gt, gte, lt, lte, make_semver
+from jupyterlab._version import __version__
 
 # The regex for expecting the webpack output.
 WEBPACK_EXPECT = re.compile(r'.*theme-light-extension/style/index.css')
 
+
+# The repo root directory
+REPO_ROOT = osp.abspath(osp.join(HERE, '..'))
+
+
 # The dev mode directory.
-DEV_DIR = osp.abspath(os.path.join(HERE, '..', 'dev_mode'))
+DEV_DIR = osp.join(REPO_ROOT, 'dev_mode')
 
 
 # If we are pinning the package, rename it `pin@<alias>`
@@ -207,8 +213,7 @@ def ensure_node_modules(cwd, logger=None):
     if ret != 0:
         yarn_proc = ProgressProcess(['node', YARN_PATH], cwd=cwd, logger=logger)
         yarn_proc.wait()
-        parent = pjoin(HERE, '..')
-        dedupe_yarn(parent, logger)
+        dedupe_yarn(REPO_ROOT, logger)
 
     return ret != 0
 
@@ -216,13 +221,12 @@ def ensure_node_modules(cwd, logger=None):
 def ensure_dev(logger=None):
     """Ensure that the dev assets are available.
     """
-    parent = pjoin(HERE, '..')
     logger = _ensure_logger(logger)
-    target = pjoin(parent, 'dev_mode', 'static')
+    target = pjoin(DEV_DIR, 'static')
 
     # Determine whether to build.
-    if ensure_node_modules(parent, logger) or not osp.exists(target):
-        yarn_proc = ProgressProcess(['node', YARN_PATH, 'build'], cwd=parent,
+    if ensure_node_modules(REPO_ROOT, logger) or not osp.exists(target):
+        yarn_proc = ProgressProcess(['node', YARN_PATH, 'build'], cwd=REPO_ROOT,
                             logger=logger)
         yarn_proc.wait()
 
@@ -267,11 +271,10 @@ def watch_packages(logger=None):
     -------
     A list of `WatchHelper` objects.
     """
-    parent = pjoin(HERE, '..')
     logger = _ensure_logger(logger)
-    ensure_node_modules(parent, logger)
+    ensure_node_modules(REPO_ROOT, logger)
 
-    ts_dir = osp.abspath(osp.join(HERE, '..', 'packages', 'metapackage'))
+    ts_dir = osp.abspath(osp.join(REPO_ROOT, 'packages', 'metapackage'))
 
     # Run typescript watch and wait for the string indicating it is done.
     ts_regex = r'.* Found 0 errors\. Watching for file changes\.'
@@ -337,6 +340,8 @@ class AppOptions(HasTraits):
 
     registry = Unicode(help="NPM packages registry URL")
 
+    splice_source = Bool(False, help="Splice source packages into app directory.")
+
     @default('logger')
     def _default_logger(self):
         return logging.getLogger('jupyterlab')
@@ -372,10 +377,8 @@ def watch(app_options=None):
 
     Parameters
     ----------
-    app_dir: string, optional
-        The application directory.
-    logger: :class:`~logger.Logger`, optional
-        The logger instance.
+    app_options: :class:`AppOptions`, optional
+        The application options.
 
     Returns
     -------
@@ -384,7 +387,13 @@ def watch(app_options=None):
     app_options = _ensure_options(app_options)
     _node_check(app_options.logger)
     handler = _AppHandler(app_options)
-    return handler.watch()
+
+    if app_options.splice_source:
+        package_procs = watch_packages(app_options.logger)
+    else:
+        package_procs = []
+
+    return package_procs + handler.watch()
 
 
 
@@ -650,6 +659,11 @@ class _AppHandler(object):
         if not production:
             minimize = False
 
+        # If splicing, make sure the source packages are built
+        if self._options.splice_source:
+            ensure_node_modules(REPO_ROOT, logger=self.logger)
+            self._run(['node', YARN_PATH, 'build:packages'], cwd=REPO_ROOT)
+
         info = ['production' if production else 'development']
         if production:
             info.append('minimized' if minimize else 'not minimized')
@@ -786,10 +800,11 @@ class _AppHandler(object):
 
         # Look for mismatched version.
         static_version = old_jlab.get('version', '')
-        core_version = old_jlab['version']
-        if Version(static_version) != Version(core_version):
-            msg = 'Version mismatch: %s (built), %s (current)'
-            return [msg % (static_version, core_version)]
+        if not static_version.endswith('-spliced'):
+            core_version = old_jlab['version']
+            if Version(static_version) != Version(core_version):
+                msg = 'Version mismatch: %s (built), %s (current)'
+                return [msg % (static_version, core_version)]
 
         shadowed_exts = self.info['shadowed_exts']
 
@@ -814,7 +829,10 @@ class _AppHandler(object):
                     messages.append('%s needs to be removed from build' % ext)
 
         # Look for mismatched dependencies
+        src_pkg_dir = pjoin(REPO_ROOT, 'packages')
         for (pkg, dep) in new_deps.items():
+            if old_deps.get(pkg, '').startswith(src_pkg_dir):
+                continue
             if pkg not in old_deps:
                 continue
             # Skip local and linked since we pick them up separately.
@@ -1157,6 +1175,14 @@ class _AppHandler(object):
         if not version:
             version = self.info['core_data']['jupyterlab']['version']
 
+        splice_source = self._options.splice_source
+        if splice_source:
+            self.logger.debug('Splicing dev packages into app directory.')
+            source_dir = DEV_DIR
+            version = __version__ + '-spliced'
+        else:
+            source_dir = pjoin(HERE, 'staging')
+
         # Look for mismatched version.
         pkg_path = pjoin(staging, 'package.json')
 
@@ -1170,8 +1196,11 @@ class _AppHandler(object):
         for fname in ['index.js', 'bootstrap.js', 'publicpath.js',
                       'webpack.config.js',
                       'webpack.prod.config.js',
-                      'webpack.prod.minimize.config.js',
-                      '.yarnrc', 'yarn.js']:
+                      'webpack.prod.minimize.config.js']:
+            target = pjoin(staging, fname)
+            shutil.copy(pjoin(source_dir, fname), target)
+
+        for fname in ['.yarnrc', 'yarn.js']:
             target = pjoin(staging, fname)
             shutil.copy(pjoin(HERE, 'staging', fname), target)
 
@@ -1181,7 +1210,7 @@ class _AppHandler(object):
             _rmtree(templates, self.logger)
 
         try:
-            shutil.copytree(pjoin(HERE, 'staging', 'templates'), templates)
+            shutil.copytree(pjoin(source_dir, 'templates'), templates)
         except shutil.Error as error:
             # `copytree` throws an error if copying to + from NFS even though
             # the copy is successful (see https://bugs.python.org/issue24564
@@ -1227,16 +1256,41 @@ class _AppHandler(object):
 
         # Then get the package template.
         data = self._get_package_template()
+        jlab = data['jupyterlab']
 
         if version:
-            data['jupyterlab']['version'] = version
+            jlab['version'] = version
 
         if name:
-            data['jupyterlab']['name'] = name
+            jlab['name'] = name
 
         if static_url:
-            data['jupyterlab']['staticUrl'] = static_url
-
+            jlab['staticUrl'] = static_url
+
+        # Handle splicing of packages
+        if splice_source:
+            # Splice workspace tree as linked dependencies
+            for path in glob(pjoin(REPO_ROOT, 'packages', '*', 'package.json')):
+                local_path = osp.dirname(osp.abspath(path))
+                pkg_data = json.loads(Path(path).read_text(encoding='utf-8'))
+                name = pkg_data['name']
+                if name in data['dependencies']:
+                    data['dependencies'][name] = local_path
+                    jlab['linkedPackages'][name] = local_path
+                if name in data['resolutions']:
+                    data['resolutions'][name] = local_path
+
+            # splice the builder as well
+            local_path = osp.abspath(pjoin(REPO_ROOT, 'builder'))
+            data['devDependencies']['@jupyterlab/builder'] = local_path
+            target = osp.join(staging, 'node_modules', '@jupyterlab', 'builder')
+
+            # Remove node_modules so it gets re-populated
+            node_modules = pjoin(staging, 'node_modules')
+            if osp.exists(node_modules):
+                shutil.rmtree(node_modules, ignore_errors=True)
+
+        # Write the package file
         pkg_path = pjoin(staging, 'package.json')
         with open(pkg_path, 'w') as fid:
             json.dump(data, fid, indent=4)
@@ -1812,7 +1866,7 @@ class _AppHandler(object):
 
             for key in keys:
                 fname = key[0].replace('@', '') + key[1:].replace('@', '-').replace('/', '-') + '.tgz'
-                data = read_package(os.path.join(tempdir, fname))
+                data = read_package(osp.join(tempdir, fname))
                 # Verify that the version is a valid extension.
                 if not _validate_extension(data):
                     # Valid
@@ -1964,10 +2018,10 @@ def _unlink(path, logger):
 def _rmtree_star(path, logger):
     """Remove all files/trees within a dir, logging errors"""
     for filename in os.listdir(path):
-        file_path = os.path.join(path, filename)
-        if os.path.isfile(file_path) or os.path.islink(file_path):
+        file_path = osp.join(path, filename)
+        if osp.isfile(file_path) or osp.islink(file_path):
             _unlink(file_path, logger)
-        elif os.path.isdir(file_path):
+        elif osp.isdir(file_path):
             _rmtree(file_path, logger)
 
 
@@ -2047,7 +2101,7 @@ def _get_static_data(app_dir):
     """Get the data for the app static dir.
     """
     target = pjoin(app_dir, 'static', 'package.json')
-    if os.path.exists(target):
+    if osp.exists(target):
         with open(target) as fid:
             return json.load(fid)
     else:

jupyterlab/labapp.py

@@ -64,6 +64,11 @@ build_flags['no-minimize'] = (
     {'LabBuildApp': {'minimize': False}},
     "Do not minimize a production build."
 )
+build_flags['splice-source'] = (
+    {'LabBuildApp': {'splice_source': True}},
+    "Splice source packages into app directory."
+)
+
 
 version = __version__
 app_version = get_app_version()
@@ -140,10 +145,13 @@ class LabBuildApp(JupyterApp, DebugLogFileMixin):
     pre_clean = Bool(False, config=True,
         help="Whether to clean before building (defaults to False)")
 
+    splice_source = Bool(False, config=True,
+        help="Splice source packages into app directory.")
+
     def start(self):
         app_dir = self.app_dir or get_app_dir()
         app_options = AppOptions(
-            app_dir=app_dir, logger=self.log, core_config=self.core_config
+            app_dir=app_dir, logger=self.log, core_config=self.core_config, splice_source=self.splice_source
         )
         self.log.info('JupyterLab %s', version)
         with self.debug_logging():
@@ -476,6 +484,10 @@ class LabApp(NBClassicConfigShimMixin, LabServerApp):
         {'LabApp': {'watch': True}},
         "Start the app in watch mode."
     )
+    flags['splice-source'] = (
+        {'LabApp': {'splice_source': True}},
+        "Splice source packages into app directory."
+    )
     flags['expose-app-in-browser'] = (
         {'LabApp': {'expose_app_in_browser': True}},
         "Expose the global app instance to browser via window.jupyterlab."
@@ -535,6 +547,9 @@ class LabApp(NBClassicConfigShimMixin, LabServerApp):
     watch = Bool(False, config=True,
         help="Whether to serve the app in watch mode")
 
+    splice_source = Bool(False, config=True,
+        help="Splice source packages into app directory.")
+
     expose_app_in_browser = Bool(False, config=True,
         help="Whether to expose the global app instance to browser via window.jupyterlab")
 
@@ -652,7 +667,7 @@ class LabApp(NBClassicConfigShimMixin, LabServerApp):
         self.log.info('JupyterLab extension loaded from %s' % HERE)
         self.log.info('JupyterLab application directory is %s' % self.app_dir)
 
-        build_handler_options = AppOptions(logger=self.log, app_dir=self.app_dir, labextensions_path = self.extra_labextensions_path + self.labextensions_path)
+        build_handler_options = AppOptions(logger=self.log, app_dir=self.app_dir, labextensions_path = self.extra_labextensions_path + self.labextensions_path, splice_source=self.splice_source)
         builder = Builder(self.core_mode, app_options=build_handler_options)
         build_handler = (build_path, BuildHandler, {'builder': builder})
         handlers.append(build_handler)
@@ -667,6 +682,8 @@ class LabApp(NBClassicConfigShimMixin, LabServerApp):
                 ensure_dev(self.log)
                 self.log.info(DEV_NOTE)
         else:
+            if self.splice_source:
+                ensure_dev(self.log)
             msgs = ensure_app(self.app_dir)
             if msgs:
                 [self.log.error(msg) for msg in msgs]

jupyterlab/labextensions.py

@@ -40,6 +40,10 @@ flags['clean'] = (
     {'BaseExtensionApp': {'should_clean': True}},
     "Cleanup intermediate files after the action."
 )
+flags['splice-source'] = (
+    {'BaseExtensionApp': {'splice_source': True}},
+    "Splice source packages into app directory."
+)
 
 check_flags = copy(flags)
 check_flags['installed'] = (
@@ -108,6 +112,9 @@ class BaseExtensionApp(JupyterApp, DebugLogFileMixin):
     should_clean = Bool(False, config=True,
         help="Whether temporary files should be cleaned up after building jupyterlab")
 
+    splice_source = Bool(False, config=True,
+        help="Splice source packages into app directory.")
+
     labextensions_path = List(Unicode(), help='The standard paths to look in for prebuilt JupyterLab extensions')
 
     @default('labextensions_path')
@@ -116,6 +123,11 @@ class BaseExtensionApp(JupyterApp, DebugLogFileMixin):
         lab.load_config_file()
         return lab.extra_labextensions_path + lab.labextensions_path
 
+    @default('splice_source')
+    def _default_splice_source(self):
+        version = get_app_version(AppOptions(app_dir=self.app_dir))
+        return version.endswith('-spliced')
+
     def start(self):
         if self.app_dir and self.app_dir.startswith(HERE):
             raise ValueError('Cannot run lab extension commands in core app')
@@ -124,7 +136,7 @@ class BaseExtensionApp(JupyterApp, DebugLogFileMixin):
             if ans and self.should_build:
                 production = None if self.dev_build is None else not self.dev_build
                 app_options = AppOptions(app_dir=self.app_dir, logger=self.log,
-                      core_config=self.core_config)
+                      core_config=self.core_config, splice_source=self.splice_source)
                 build(clean_staging=self.should_clean,
                       production = production, minimize = self.minimize, app_options=app_options)
 

jupyterlab/staging/webpack.config.js

@@ -87,7 +87,12 @@ Object.keys(jlab.linkedPackages).forEach(function (name) {
   if (name in watched) {
     return;
   }
-  const localPkgPath = require.resolve(path.join(name, 'package.json'));
+  let localPkgPath = '';
+  try {
+    localPkgPath = require.resolve(path.join(name, 'package.json'));
+  } catch (e) {
+    return;
+  }
   watched[name] = path.dirname(localPkgPath);
   if (localPkgPath.indexOf('node_modules') !== -1) {
     watchNodeModules = true;
@@ -106,7 +111,13 @@ const sourceMapRes = Object.values(watched).reduce((res, name) => {
  * and has no effect in `jupyter lab --dev-mode --watch`.
  */
 function maybeSync(localPath, name, rest) {
-  const stats = fs.statSync(localPath);
+  let stats;
+  try {
+    stats = fs.statSync(localPath);
+  } catch (e) {
+    return;
+  }
+
   if (!stats.isFile(localPath)) {
     return;
   }

jupyterlab/tests/test_jupyterlab.py

@@ -26,7 +26,7 @@ from jupyterlab.commands import (
     disable_extension, enable_extension,
     get_app_info, install_extension, link_package,
     list_extensions, uninstall_extension,
-    unlink_package, update_extension
+    unlink_package, update_extension, get_app_version
 )
 from jupyterlab.coreconfig import CoreConfig, _get_default_core_data
 
@@ -447,6 +447,24 @@ class TestExtension(AppHandlerTest):
             data = fid.read()
         assert self.pkg_names['extension'] in data
 
+    @pytest.mark.slow
+    def test_build_splice_packages(self):
+        app_options = AppOptions(splice_source=True)
+        assert install_extension(self.mock_extension) is True
+        build(app_options=app_options)
+        assert '-spliced' in get_app_version(app_options)
+        # check staging directory.
+        entry = pjoin(self.app_dir, 'staging', 'build', 'index.out.js')
+        with open(entry) as fid:
+            data = fid.read()
+        assert self.pkg_names['extension'] in data
+
+        # check static directory.
+        entry = pjoin(self.app_dir, 'static', 'index.out.js')
+        with open(entry) as fid:
+            data = fid.read()
+        assert self.pkg_names['extension'] in data
+
     @pytest.mark.slow
     def test_build_custom(self):
         assert install_extension(self.mock_extension) is True

scripts/ci_script.sh

@@ -362,6 +362,37 @@ if [[ $GROUP == usage2 ]]; then
     jupyter lab clean --all
 fi
 
+
+if [[ $GROUP == splice_source ]];then
+    # Run the integrity script to link binary files
+    jlpm integrity
+
+    jupyter lab build --minimize=False --debug --dev-build=True --splice-source
+    jupyter lab --version > version.txt
+    cat version.txt
+    cat version.txt | grep -q "spliced"
+    python -m jupyterlab.browser_check
+
+    cd jupyterlab/tests/mock_packages/mimeextension
+    jupyter labextension install .
+    python -m jupyterlab.browser_check
+
+    jupyter lab --version > version.txt
+    cat version.txt
+    cat version.txt | grep -q "spliced"
+
+    jupyter lab clean --all
+    jupyter lab --version > version.txt
+    cat version.txt
+    cat version.txt | grep -q "spliced" && exit 1
+
+    jupyter labextension install --splice-source .
+    jupyter lab --version > version.txt
+    cat version.txt | grep -q "spliced"
+    python -m jupyterlab.browser_check
+fi
+
+
 if [[ $GROUP == interop ]]; then
     cd jupyterlab/tests/mock_packages/interop