wip clean up extension install and upgrade

jupyterlab/dynamic_labextensions.py

@@ -198,16 +198,15 @@ def watch_labextension(path, labextensions_path, logger=None, development=False,
 
     with open(pjoin(ext_path, 'package.json')) as fid:
         ext_data = json.load(fid)
-    
-    os.makedirs(ext_data['jupyterlab']['outputDir'], exist_ok=True)
 
     if ext_data['name'] not in dynamic_exts:
-        full_dest = develop_labextension(ext_path, sys_prefix=True)
+        full_dest = develop_labextension_py(ext_path, sys_prefix=True)
     else:
         full_dest = pjoin(dynamic_exts[ext_data['name']]['ext_dir'], ext_data['name'])
+        output_dir = pjoin(ext_path, ext_data['jupyterlab']['outputDir'])
         if not osp.islink(full_dest):
             shutil.rmtree(full_dest)
-            os.symlink(ext_path, full_dest)
+            os.symlink(output_dir, full_dest)
 
     builder = _ensure_builder(ext_path, core_path)
     arguments = ['node', builder, '--core-path', core_path,  '--watch', ext_path]
@@ -363,7 +362,6 @@ def _get_labextension_metadata(module):
     if not hasattr(m, '_jupyter_labextension_paths'):
         mod_path = osp.abspath(module)
         if osp.exists(mod_path):
-            sys.path.insert(0, mod_path)
 
             # First see if the module is already installed
             from setuptools import find_packages
@@ -374,10 +372,17 @@ def _get_labextension_metadata(module):
                 name = subprocess.check_output([sys.executable, 'setup.py', '--name'], cwd=mod_path)
                 packages = [name.decode('utf8').strip()]
             
-            # Import the first found package
-            m = import_item(packages[0])
+            package = packages[0]
+
+            # Make sure the package is installed
+            import pkg_resources
+            try:
+                dist = pkg_resources.get_distribution(package)
+            except pkg_resources.DistributionNotFound:
+                subprocess.check_call([sys.executable, '-m', 'pip', 'install', '-e', mod_path])
+                sys.path.insert(0, mod_path)
 
-            sys.path.pop(0)
+            m = import_item(package)
 
     if not hasattr(m, '_jupyter_labextension_paths'):
         raise KeyError('The Python module {} is not a valid labextension, '

jupyterlab/upgrade_extension.py

@@ -8,7 +8,7 @@ import sys
 import subprocess
 
 try:
-    import cookiecutter
+    from cookiecutter.main import cookiecutter
 except ImportError:
     raise RuntimeError("Please install cookiecutter")
 
@@ -16,6 +16,25 @@ except ImportError:
 COOKIECUTTER_BRANCH = "3.0"
 
 
+"""TODO
+[x] use the cookiecuter api directly
+[x] do no overwrite files when --no-input
+[x] make sure the name is all hypen -> underscore for python name
+[x] fix devDependencies
+[x] make sure lint still works
+[x] make sure `yarn && yarn watch` works
+[x] if not installed, run pip develop install
+[x] the symlink is in the wrong directory when running `pip install -e .` and then watching
+[ ] update cookiecutter
+[ ] test all of the advanced extensions
+[ ] test on an example with a server extension
+[ ] make a commit where we apply the auto changes to server ext
+[ ] make another commit where we apply the manual extensiosn
+[ ] write a higher-level script that goes across the whole repo
+   - leaves server extension out 
+   - gives a top level summary of folders that had setup.py in them
+"""
+
 def update_extension(target, interactive=True):
     """Update an extension to the current JupyterLab
 
@@ -45,9 +64,14 @@ def update_extension(target, interactive=True):
     else:
         python_name = data['name']
         if '@' in python_name:
-            python_name = python_name[1:].replace('/', '_')
+            python_name = python_name[1:].replace('/', '_').replace('-', '_')
     
-    arg_data = dict(
+    output_dir = osp.join(target, '_temp_extension')
+    if osp.exists(output_dir):
+        shutil.rmtree(output_dir)
+
+    # Build up the cookiecutter args and run the cookiecutter
+    extra_context = dict(
         author_name = data.get('author', '<author_name>'),
         labextension_name = data['name'],
         project_short_description = data.get('description', '<description>'),
@@ -57,20 +81,17 @@ def update_extension(target, interactive=True):
         python_name = python_name
     )
 
-    args = ['%s=%s' % (key, value) for (key, value) in arg_data.items()]
-    repo = 'https://github.com/jupyterlab/extension-cookiecutter-ts'
-
-    extension_dir = osp.join(target, '_temp_extension')
-    if osp.exists(extension_dir):
-        shutil.rmtree(extension_dir)
-
-    if not interactive:
-        args.append('--no-input')
+    template = 'https://github.com/jupyterlab/extension-cookiecutter-ts'
+    cookiecutter(template=template, checkout=COOKIECUTTER_BRANCH, output_dir=output_dir, 
+                extra_context=extra_context, no_input=not interactive)
 
-    subprocess.run(['cookiecutter', repo, '--checkout', COOKIECUTTER_BRANCH, '-o', extension_dir] + args, cwd=target)
+    python_name = os.listdir(output_dir)[0]
 
-    python_name = os.listdir(extension_dir)[0]
-    extension_dir = osp.join(extension_dir, python_name)
+    # hoist the output up one level
+    shutil.move(osp.join(output_dir, python_name), osp.join(output_dir, '_temp'))
+    for filename in os.listdir(osp.join(output_dir, '_temp')):
+        shutil.move(osp.join(output_dir, '_temp', filename), osp.join(output_dir, filename))
+    shutil.rmtree(osp.join(output_dir, '_temp'))
     
     # Check whether there are any phosphor dependencies
     has_phosphor = False
@@ -81,14 +102,14 @@ def update_extension(target, interactive=True):
         for (key, value) in data[name].items():
             if key.startswith('@phosphor/'):
                 has_phosphor = True
-            data[key.replace('@phosphor/', '@lumino/')] = value
+                data[name][key.replace('@phosphor/', '@lumino/')] = value
         
         for key in list(data[name]):
             if key.startswith('@phosphor/'):
                 del data[name][key]
 
     # From the created package.json grab the devDependencies
-    with open(osp.join(extension_dir, 'package.json')) as fid:
+    with open(osp.join(output_dir, 'package.json')) as fid:
         temp_data = json.load(fid)
 
     for (key, value) in temp_data['devDependencies'].items():
@@ -139,7 +160,7 @@ def update_extension(target, interactive=True):
 
     # For the other files, ask about whether to override (when it exists)
     # At the end, list the files that were: added, overridden, skipped
-    path = Path(extension_dir)
+    path = Path(output_dir)
     for p in path.rglob("*"):
         relpath = osp.relpath(p, path)
         if relpath == "package.json":
@@ -160,11 +181,11 @@ def update_extension(target, interactive=True):
             if interactive:
                 choice = input('overwrite "%s"? [n]: ' % relpath)
             else:
-                choice = 'y'
+                choice = 'n'
             if choice.upper().startswith('Y'):
                 shutil.copy(p, file_target)
             else:
-                warnings.append('skipped %s' % relpath)
+                warnings.append('skipped _temp_extension/%s' % relpath)
 
     # Print out all warnings
     for warning in warnings:

setup.py

@@ -153,7 +153,7 @@ setup_args = dict(
 setup_args['install_requires'] = [
     'ipython',
     'tornado!=6.0.0, !=6.0.1, !=6.0.2',
-    'jupyterlab_server~=2.0.0b7',
+    'jupyterlab_server~=2.0.0b8',
     'jupyter_server~=1.0.0rc16',
     'nbclassic~=0.2.0rc7',
     'jinja2>=2.10'