Adding testing of a symlink tree install

.github/workflows/test.yml

@@ -100,8 +100,12 @@ jobs:
           cat test_output
           grep --quiet 'Python 3.9.5' test_output
           rm test_output
-          shpc uninstall --force python:3.9.5-alpine 
+          shpc uninstall --force python:3.9.5-alpine
 
+          # Try creating symlink install
+          mkdir -p tmp-modules
+          shpc config set symlink_base:tmp-modules
+          shpc install python:3.9.5-alpine --symlink-tree
 
       - name: Run python module tests (tcsh)
         shell: tcsh -e {0}

CHANGELOG.md

@@ -14,6 +14,9 @@ and **Merged pull requests**. Critical items to know are:
 The versions coincide with releases on pip. Only major versions will be released as tags on Github.
 
 ## [0.0.x](https://github.com/singularityhub/singularity-hpc/tree/main) (0.0.x)
+ - support for installing to symlink tree (0.0.46)
+  - also including cleanup of symlink tree on uninstall
+  - ability to set custom config variable on the fly with -c
  - Adding support for wrapper scripts for global and container.yaml (0.0.45)
  - Lua and tcl module file bug fixes for shell functions and aliases (0.0.44)
   - restoring -B for Singularity bindpaths over using envar

docs/conf.py

@@ -290,4 +290,4 @@
 
 
 def setup(app):
-    app.add_stylesheet("sphinx-argparse.css")
+    app.add_css_file("sphinx-argparse.css")

docs/getting_started/user-guide.rst

@@ -52,6 +52,11 @@ You can then easily install, load, and use modules:
     $ module load biocontainers/samtools
     $ samtools
 
+Or set a configuration value on the fly for any command:
+
+.. code-block:: console
+
+    $ shpc install -c set:symlink_base:/tmp/modules biocontainers/samtools
 
 The above assumes that you've installed the software, and have already
 added the modules folder to be seen by your module software. If your module
@@ -174,6 +179,12 @@ variable replacement. A summary table of variables is included below, and then f
    * - container_tech
      - The container technology to use (singularity or podman)
      - singularity
+   * - symlink_base
+     - If set, where you want to install a simplified module tree to using ``--symlink-tree``
+     - unset
+   * - symlink_tree
+     - If set to true, ALWAYS generate a symlink tree given that a symlink base is defined regardless of ``--symlink-tree`` flag
+     - false
    * - updated_at
      - a timestamp to keep track of when you last saved
      - never
@@ -233,6 +244,14 @@ variable replacement. A summary table of variables is included below, and then f
      - All features default to null
 
 
+Note that any configuration value can be set permanently by using ``shpc config``
+or manually editing the file, but you can also set config values "one off" as follows:
+
+.. code-block:: console
+
+    $ shpc install -c set:symlink_base:/tmp/modules ghcr.io/autamus/clingo
+
+
 These settings will be discussed in more detail in the following sections.
 
 Features
@@ -359,6 +378,57 @@ you can add or remove entries via the config variable ``registry``
 # Note that "add" is used for lists of things (e.g., the registry config variable is a list)
 and "set" is used to set a key value pair.
 
+Symlink Base
+------------
+
+By default, your modules are installed to your ``module_base`` described above with a complete
+namespace, meaning the container registry from where they arise. We do this so that the namespace
+is consistent and there are no conflicts. However, if you want a simplified tree to install from,
+meaning the module full names are _just_ the final container name, you can set the ``symlink_base``
+in your settings to a different root. For example, let's say we want to install a set of modules,
+after seting our symlink base to ``tmp-modules``. We could do:
+
+.. code-block:: console
+
+    $ shpc install ghcr.io/autamus/clingo --symlink-tree
+    $ shpc install ghcr.io/autamus/samtools --symlink-tree
+
+Then, for example, if you want to load the modules, you'll see the shorter names are
+available!
+
+.. code-block:: console
+
+    $ module use ./tmp-modules
+    $ module load clingo/5.5.1/module
+
+This is much more efficient compared to the install that uses the full paths:
+
+.. code-block:: console
+
+    $ module use ./modules
+    $ module load ghcr.io/autamus/clingo/5.5.1/module
+
+Since we install based on the container name *and* version tag, this even gives you
+the ability to install versions from different container bases in the same root.
+If there is a conflict, you will be given the option to exit (and abort) or continue.
+Finally, if you need an easy way to run through the containers you've already installed
+to create the links:
+
+
+.. code-block:: console
+
+    for module in $(shpc list); do
+        shpc install $module --symlink-tree
+    done
+
+And that will reinstall the modules you have installed, but in their symlink tree location.
+
+
+.. warning::
+
+    Be cautious about creating symlinks in containers or other contexts where a bind
+    could eliminate the symlink or make the path non-existent.
+
 
 Module Names
 ------------

shpc/client/__init__.py

@@ -40,6 +40,17 @@ def get_parser():
         help="custom path to settings file.",
     )
 
+    # On the fly updates to config params
+    parser.add_argument(
+        "-c",
+        dest="config_params",
+        help=""""customize a config value on the fly to ADD/SET/REMOVE for a command
+shpc -c set:key:value <command> <args>
+shpc -c add:registry:/tmp/registry <command> <args>
+shpc -c rm:registry:/tmp/registry""",
+        action="append",
+    )
+
     parser.add_argument(
         "--version",
         dest="version",
@@ -89,6 +100,13 @@ def get_parser():
         "install_recipe",
         help="recipe to install\nshpc install python\nshpc install python:3.9.5-alpine",
     )
+    install.add_argument(
+        "--symlink-tree",
+        dest="symlink",
+        help="install to symlink tree too.",
+        default=False,
+        action="store_true",
+    )
 
     # List installed modules
     listing = subparsers.add_parser("list", description="list installed modules.")
@@ -147,7 +165,6 @@ def get_parser():
 
     config.add_argument(
         "--central",
-        "-c",
         dest="central",
         help="make edits to the central config file.",
         default=False,

shpc/client/add.py

@@ -13,4 +13,7 @@ def main(args, parser, extra, subparser):
         module=args.module,
         container_tech=args.container_tech,
     )
+
+    # Update config settings on the fly
+    cli.settings.update_params(args.config_params)
     cli.add(args.sif_path[0], args.module_id[0])

shpc/client/check.py

@@ -13,4 +13,7 @@ def main(args, parser, extra, subparser):
         module=args.module,
         container_tech=args.container_tech,
     )
+
+    # Update config settings on the fly
+    cli.settings.update_params(args.config_params)
     cli.check(args.module_name)

shpc/client/config.py

@@ -33,25 +33,12 @@ def main(args, parser, extra, subparser):
         return cli.settings.inituser()
     if command == "edit":
         return cli.settings.edit()
-    elif command in ["set", "add", "remove"]:
-        for param in args.params:
-            if ":" not in param:
-                logger.warning(
-                    "Param %s is missing a :, should be key:value pair. Skipping."
-                    % param
-                )
-                continue
 
-            key, value = param.split(":", 1)
-            if command == "set":
-                cli.settings.set(key, value)
-                logger.info("Updated %s to be %s" % (key, value))
-            elif command == "add":
-                cli.settings.add(key, value)
-                logger.info("Added %s to %s" % (key, value))
-            elif command == "remove":
-                cli.settings.remove(key, value)
-                logger.info("Removed %s from %s" % (key, value))
+    if command in ["set", "add", "remove"]:
+
+        # Update each param
+        for param in args.params:
+            cli.settings.update_param(command, param)
 
         # Save settings
         cli.settings.save()

shpc/client/docgen.py

@@ -13,4 +13,6 @@ def main(args, parser, extra, subparser):
         module=args.module,
         container_tech=args.container_tech,
     )
+    # Update config settings on the fly
+    cli.settings.update_params(args.config_params)
     cli.docgen(args.module_name)

shpc/client/get.py

@@ -14,6 +14,8 @@ def main(args, parser, extra, subparser):
         container_tech=args.container_tech,
     )
 
+    # Update config settings on the fly
+    cli.settings.update_params(args.config_params)
     result = cli.get(args.module_name, args.env_file)
     if result:
         print(result)

shpc/client/inspect.py

@@ -15,6 +15,9 @@ def main(args, parser, extra, subparser):
         settings_file=args.settings_file,
         container_tech=args.container_tech,
     )
+
+    # Update config settings on the fly
+    cli.settings.update_params(args.config_params)
     metadata = cli.inspect(args.module_name)
 
     # Case 1: dump entire thing as json

shpc/client/install.py

@@ -16,4 +16,9 @@ def main(args, parser, extra, subparser):
         module=args.module,
         container_tech=args.container_tech,
     )
-    cli.install(args.install_recipe)
+
+    # Update config settings on the fly
+    cli.settings.update_params(args.config_params)
+
+    # And do the install
+    cli.install(args.install_recipe, symlink=args.symlink)

shpc/client/listing.py

@@ -14,4 +14,6 @@ def main(args, parser, extra, subparser):
         container_tech=args.container_tech,
     )
 
+    # Update config settings on the fly
+    cli.settings.update_params(args.config_params)
     cli.list(args.pattern, args.names_only, short=args.short)

shpc/client/namespace.py

@@ -13,6 +13,9 @@ def main(args, parser, extra, subparser):
 
     cli = get_client(quiet=args.quiet, settings_file=args.settings_file)
 
+    # Update config settings on the fly
+    cli.settings.update_params(args.config_params)
+
     # Case 1: we need to unset a namespace
     if not args.namespace:
         sys.exit("Please choose: shpc use <namespace> or shpc unset.")

shpc/client/shell.py

@@ -34,13 +34,17 @@ def main(args, parser, extra, subparser):
 def create_client(args):
     from shpc.main import get_client
 
-    return get_client(
+    cli = get_client(
         quiet=args.quiet,
         settings_file=args.settings_file,
         module=args.module,
         container_tech=args.container_tech,
     )
 
+    # Update config settings on the fly
+    cli.settings.update_params(args.config_params)
+    return cli
+
 
 def ipython(args):
     """

shpc/client/show.py

@@ -8,4 +8,7 @@ def main(args, parser, extra, subparser):
     from shpc.main import get_client
 
     cli = get_client(quiet=args.quiet, settings_file=args.settings_file)
+
+    # Update config settings on the fly
+    cli.settings.update_params(args.config_params)
     cli.show(args.name, names_only=not args.versions, filter_string=args.filter_string)

shpc/client/test.py

@@ -14,6 +14,8 @@ def main(args, parser, extra, subparser):
         container_tech=args.container_tech,
     )
 
+    # Update config settings on the fly
+    cli.settings.update_params(args.config_params)
     cli.test(
         args.module_name,
         test_exec=args.test_exec,

shpc/client/uninstall.py

@@ -13,4 +13,7 @@ def main(args, parser, extra, subparser):
         module=args.module,
         container_tech=args.container_tech,
     )
+
+    # Update config settings on the fly
+    cli.settings.update_params(args.config_params)
     cli.uninstall(args.uninstall_recipe, args.force)