Merge pull request #8913 from jasongrout/caching

Pull dynamic extension loading data from the webpack compilation

builder/src/webpack.config.ext.ts

@@ -6,7 +6,9 @@ import * as webpack from 'webpack';
 import { Build } from './build';
 import { merge } from 'webpack-merge';
 import * as fs from 'fs';
+import * as glob from 'glob';
 import Ajv from 'ajv';
+import { readJSONFile, writeJSONFile } from '@jupyterlab/buildutils';
 
 const baseConfig = require('./webpack.config.base');
 const { ModuleFederationPlugin } = webpack.container;
@@ -142,13 +144,9 @@ const extras = Build.ensureAssets({
 
 fs.copyFileSync(
   path.join(packagePath, 'package.json'),
-  path.join(outputPath, 'package.orig.json')
+  path.join(outputPath, 'package.json')
 );
 
-// TODO: We don't need this file after our compilation, since it is folded
-// into remoteEntry.js. We should either delete it, or figure out a way to
-// have the entry point below be dynamically generated text without having to
-// write to a file.
 const webpackPublicPathString = staticPath
   ? `"${staticPath}"`
   : `getOption('fullLabextensionsUrl') + '/${data.name}/'`;
@@ -174,6 +172,34 @@ __webpack_public_path__ = ${webpackPublicPathString};
 `
 );
 
+class CleanupPlugin {
+  apply(compiler: any) {
+    compiler.hooks.done.tap('Cleanup', () => {
+      fs.unlinkSync(publicpath);
+      // Find the remoteEntry file and add it to the package.json metadata
+      const files = glob.sync(path.join(outputPath, 'remoteEntry.*.js'));
+      if (files.length !== 1) {
+        throw new Error('There is not a single remoteEntry file generated.');
+      }
+      const data = readJSONFile(path.join(outputPath, 'package.json'));
+      const _build: any = {
+        load: path.basename(files[0])
+      };
+      if (exposes['./extension'] !== undefined) {
+        _build.extension = './extension';
+      }
+      if (exposes['./mimeExtension'] !== undefined) {
+        _build.mimeExtension = './mimeExtension';
+      }
+      if (exposes['./style'] !== undefined) {
+        _build.style = './style';
+      }
+      data.jupyterlab._build = _build;
+      writeJSONFile(path.join(outputPath, 'package.json'), data);
+    });
+  }
+}
+
 module.exports = [
   merge(baseConfig, {
     // Using empty object {} for entry because we want only
@@ -182,7 +208,7 @@ module.exports = [
       [data.name]: publicpath
     },
     output: {
-      filename: '[name].[chunkhash].js',
+      filename: '[name].[contenthash].js',
       path: outputPath
     },
     module: {
@@ -195,13 +221,14 @@ module.exports = [
           type: 'var',
           name: ['_JUPYTERLAB', data.name]
         },
-        filename: 'remoteEntry.js',
+        filename: 'remoteEntry.[contenthash].js',
         exposes,
         shared
-      })
+      }),
+      new CleanupPlugin()
     ]
   })
 ].concat(extras);
 
-const logPath = path.join(outputPath, 'build_log.json');
-fs.writeFileSync(logPath, JSON.stringify(module.exports, null, '  '));
+// const logPath = path.join(outputPath, 'build_log.json');
+// fs.writeFileSync(logPath, JSON.stringify(module.exports, null, '  '));

dev_mode/index.js

@@ -81,8 +81,8 @@ async function main() {
     PageConfig.getOption('dynamic_extensions')
   );
 
-  const dynamicPluginPromises = [];
-  const dynamicMimePluginPromises = [];
+  const dynamicExtensionPromises = [];
+  const dynamicMimeExtensionPromises = [];
   const dynamicStylePromises = [];
 
   // We first load all dynamic components so that the shared module
@@ -90,7 +90,7 @@ async function main() {
   // components should be actually used.
   const extensions = await Promise.allSettled(extension_data.map( async data => {
     await loadComponent(
-      `${URLExt.join(PageConfig.getOption('fullLabextensionsUrl'), data.name, 'remoteEntry.js')}`,
+      `${URLExt.join(PageConfig.getOption('fullLabextensionsUrl'), data.name, data.load)}`,
       data.name
     );
     return data;
@@ -104,11 +104,11 @@ async function main() {
     }
 
     const data = p.value;
-    if (data.plugin) {
-      dynamicPluginPromises.push(createModule(data.name, data.plugin));
+    if (data.extension) {
+      dynamicExtensionPromises.push(createModule(data.name, data.extension));
     }
-    if (data.mimePlugin) {
-      dynamicMimePluginPromises.push(createModule(data.name, data.mimePlugin));
+    if (data.mimeExtension) {
+      dynamicMimeExtensionPromises.push(createModule(data.name, data.mimeExtension));
     }
     if (data.style) {
       dynamicStylePromises.push(createModule(data.name, data.style));
@@ -158,8 +158,8 @@ async function main() {
   {{/each}}
 
   // Add the dynamic mime extensions.
-  const dynamicMimePlugins = await Promise.allSettled(dynamicMimePluginPromises);
-  dynamicMimePlugins.forEach(p => {
+  const dynamicMimeExtensions = await Promise.allSettled(dynamicMimeExtensionPromises);
+  dynamicMimeExtensions.forEach(p => {
     if (p.status === "fulfilled") {
       for (let plugin of activePlugins(p.value)) {
         mimeExtensions.push(plugin);
@@ -181,8 +181,8 @@ async function main() {
   {{/each}}
 
   // Add the dynamic extensions.
-  const dynamicPlugins = await Promise.allSettled(dynamicPluginPromises);
-  dynamicPlugins.forEach(p => {
+  const dynamicExtensions = await Promise.allSettled(dynamicExtensionPromises);
+  dynamicExtensions.forEach(p => {
     if (p.status === "fulfilled") {
       for (let plugin of activePlugins(p.value)) {
         register.push(plugin);

dev_mode/webpack.config.js

@@ -192,7 +192,6 @@ const plugins = [
     template: plib.join(__dirname, 'templates', 'template.html'),
     title: jlab.name || 'JupyterLab'
   }),
-  new webpack.ids.HashedModuleIdsPlugin(),
   // custom plugin for ignoring files during a `--watch` build
   new WPPlugin.FilterWatchIgnorePlugin(ignored),
   // custom plugin that copies the assets to the static directory
@@ -223,7 +222,7 @@ module.exports = [
     output: {
       path: plib.resolve(buildDir),
       publicPath: '{{page_config.fullStaticUrl}}/',
-      filename: '[name].[chunkhash].js'
+      filename: '[name].[contenthash].js'
     },
     optimization: {
       splitChunks: {

docs/source/developer/extension_dev.rst

@@ -38,9 +38,8 @@ Implementation
 - We provide a ``jupyter labextensions build`` script that is used to build bundles
   - The command produces a set of static assets that are shipped along with a package (notionally on ``pip``/``conda``)
   - It needs to be a Python cli so it can use the dependency metadata from the active JupyterLab
-  - The assets include a module federation ``remoteEntry.js``, generated bundles, and some other files that we use
-  - ``package.orig.json`` is the original ``package.json`` file that we use to gather metadata about the package
-  - ``build_log.json`` has all of the webpack options used to build the extension, for debugging purposes
+  - The assets include a module federation ``remoteEntry.*.js``, generated bundles, and some other files that we use
+  - ``package.json`` is the original ``package.json`` file that we use to gather metadata about the package, with some included build metadata
   - we use the existing ``@jupyterlab/builder -> build`` to generate the ``imports.css``, ``schemas`` and ``themes`` file structure
 - We add a schema for the valid ``jupyterlab`` metadata for an extension's ``package.json`` describing the available options
 - We add a ``labextensions`` handler in ``jupyterlab_server`` that loads static assets from ``labextensions`` paths, following a similar logic to how ``nbextensions`` are discovered and loaded from disk

examples/federated/core_package/index.js

@@ -72,8 +72,8 @@ async function main() {
     PageConfig.getOption('dynamic_extensions')
   );
 
-  const dynamicPluginPromises = [];
-  const dynamicMimePluginPromises = [];
+  const dynamicExtensionPromises = [];
+  const dynamicMimeExtensionPromises = [];
   const dynamicStylePromises = [];
 
   // We first load all dynamic components so that the shared module
@@ -81,7 +81,7 @@ async function main() {
   // components should be actually used.
   const extensions = await Promise.allSettled(extension_data.map( async data => {
     await loadComponent(
-      `${URLExt.join(PageConfig.getOption('fullLabextensionsUrl'), data.name, 'remoteEntry.js')}`,
+      `${URLExt.join(PageConfig.getOption('fullLabextensionsUrl'), data.name, data.load)}`,
       data.name
     );
     return data;
@@ -95,11 +95,11 @@ async function main() {
     }
 
     const data = p.value;
-    if (data.plugin) {
-      dynamicPluginPromises.push(createModule(data.name, data.plugin));
+    if (data.extension) {
+      dynamicExtensionPromises.push(createModule(data.name, data.extension));
     }
-    if (data.mimePlugin) {
-      dynamicMimePluginPromises.push(createModule(data.name, data.mimePlugin));
+    if (data.mimeExtension) {
+      dynamicMimeExtensionPromises.push(createModule(data.name, data.mimeExtension));
     }
     if (data.style) {
       dynamicStylePromises.push(createModule(data.name, data.style));
@@ -149,8 +149,8 @@ async function main() {
   {{/each}}
 
   // Add the dynamic mime extensions.
-  const dynamicMimePlugins = await Promise.allSettled(dynamicMimePluginPromises);
-  dynamicMimePlugins.forEach(p => {
+  const dynamicMimeExtensions = await Promise.allSettled(dynamicMimeExtensionPromises);
+  dynamicMimeExtensions.forEach(p => {
     if (p.status === "fulfilled") {
       for (let plugin of activePlugins(p.value)) {
         mimeExtensions.push(plugin);
@@ -172,8 +172,8 @@ async function main() {
   {{/each}}
 
   // Add the dynamic extensions.
-  const dynamicPlugins = await Promise.allSettled(dynamicPluginPromises);
-  dynamicPlugins.forEach(p => {
+  const dynamicExtensions = await Promise.allSettled(dynamicExtensionPromises);
+  dynamicExtensions.forEach(p => {
     if (p.status === "fulfilled") {
       for (let plugin of activePlugins(p.value)) {
         register.push(plugin);

examples/federated/main.py

@@ -6,6 +6,7 @@ from jupyterlab_server.server import FileFindHandler
 import json
 import os
 from traitlets import Unicode
+from glob import glob
 
 HERE = os.path.abspath(os.path.dirname(__file__))
 
@@ -48,19 +49,25 @@ class ExampleApp(LabServerApp):
         # By default, make terminals available.
         web_app.settings.setdefault('terminals_available', True)
 
-        # Add labextension metadata
-        page_config['dynamic_extensions'] = [{
-            'name': '@jupyterlab/example-federated-md',
-            'plugin': './extension',
-            'mimePlugin': './mimeExtension',
-            'style': './style'
-        }, {
-            'name': '@jupyterlab/example-federated-middle',
-            'plugin': './extension'
-        }, {
-            'name': '@jupyterlab/example-federated-phosphor',
-            'plugin': './index'
-        }]
+        # Extract the dynamic extension data from lab_extensions
+        dynamic_exts = []
+        for ext_path in [path for path in glob('./labextensions/**/package.json', recursive=True)]:
+            with open(ext_path) as fid:
+                data = json.load(fid)
+            extbuild = data['jupyterlab']['_build']
+            ext = {
+                'name': data['name'],
+                'load': extbuild['load'],
+            }
+            if 'extension' in extbuild:
+                ext['extension'] = extbuild['extension']
+            if 'mimeExtension' in extbuild:
+                ext['mimeExtension'] = extbuild['mimeExtension']
+            if 'style' in extbuild:
+                ext['style'] = extbuild['style']
+            dynamic_exts.append(ext)
+
+        page_config['dynamic_extensions'] = dynamic_exts
 
         super().initialize_handlers()
 

jupyterlab/commands.py

@@ -1110,7 +1110,7 @@ class _AppHandler(object):
         dynamic_exts = dict()
         dynamic_ext_dirs = dict()
         for ext_dir in jupyter_path('labextensions'):
-            ext_pattern = ext_dir + '/**/package.orig.json'
+            ext_pattern = ext_dir + '/**/package.json'
             for ext_path in [path for path in glob(ext_pattern, recursive=True)]:
                 with open(ext_path) as fid:
                     data = json.load(fid)

jupyterlab/labapp.py

@@ -709,15 +709,17 @@ class LabApp(NBClassicConfigShimMixin, LabServerApp):
         info = get_app_info()
         extensions = page_config['dynamic_extensions'] = []
         for (ext, ext_data) in info.get('dynamic_exts', dict()).items():
+            extbuild = ext_data['_build']
             extension = {
-                'name': ext_data['name']
+                'name': ext_data['name'],
+                'load': extbuild['load']
             }
-            if ext_data['jupyterlab'].get('extension'):
-                extension['plugin'] = './extension'
-            if ext_data['jupyterlab'].get('mimeExtension'):
-                extension['mimePlugin'] = './mimeExtension'
-            if ext_data.get('style'):
-                extension['style'] = './style'
+            if 'extension' in extbuild:
+                extension['extension'] = extbuild['extension']
+            if 'mimeExtension' in extbuild:
+                extension['mimeExtension'] = extbuild['mimeExtension']
+            if 'style' in extbuild:
+                extension['style'] = extbuild['style']
             extensions.append(extension)
 
         # Update Jupyter Server's webapp settings with jupyterlab settings.