diff --git a/lib/wit/main.py b/lib/wit/main.py index 7724b44..0aa32a0 100644 --- a/lib/wit/main.py +++ b/lib/wit/main.py @@ -18,7 +18,6 @@ from .workspace import WorkSpace, PackageNotInWorkspaceError from .dependency import parse_dependency_tag, Dependency from .inspect import inspect_tree -from . import scalaplugin from pathlib import Path from typing import cast, List, Tuple # noqa: F401 from .common import error, WitUserError, print_errors @@ -34,9 +33,12 @@ class NotAPackageError(WitUserError): pass -def main() -> None: - # Parse arguments. Create sub-commands for each of the modes of operation - parser = argparse.ArgumentParser(formatter_class=argparse.RawTextHelpFormatter) +class ExpandPath(argparse.Action): + def __call__(self, parser, namespace, values, option_string=None): + setattr(namespace, self.dest, os.path.abspath(os.path.expanduser(values))) + + +def build_base_parser(parser): parser.add_argument('-v', '--verbose', action='count', default=0, help='''Specify level of verbosity -v: verbose @@ -45,19 +47,22 @@ def main() -> None: -vvvv: spam ''') parser.add_argument('--version', action='store_true', help='Print wit version') - parser.add_argument('-C', dest='cwd', type=chdir, metavar='path', help='Run in given path') + parser.add_argument('-C', dest='cwd', metavar='path', + help='Run in given path', action=ExpandPath) parser.add_argument('--repo-path', default=os.environ.get('WIT_REPO_PATH'), help='Specify alternative paths to look for packages') parser.add_argument('--prepend-repo-path', default=None, help='Prepend paths to the default repo search path.') + +def add_sub_parsers(parser): subparsers = parser.add_subparsers(dest='command', help='sub-command help') init_parser = subparsers.add_parser('init', help='create workspace') init_parser.add_argument('--no-update', dest='update', action='store_false', help=('don\'t run update upon creating the workspace' ' (implies --no-fetch-scala)')) - init_parser.add_argument('--no-fetch-scala', dest='fetch_scala', action='store_false', + init_parser.add_argument('--no-fetch-scala', dest='no_fetch_scala', action='store_true', help='don\'t run fetch-scala upon creating the workspace') init_parser.add_argument('-a', '--add-pkg', metavar='repo[::revision]', action='append', type=parse_dependency_tag, help='add an initial package') @@ -85,9 +90,24 @@ def main() -> None: inspect_group.add_argument('--tree', action="store_true") inspect_group.add_argument('--dot', action="store_true") - subparsers.add_parser('fetch-scala', help='Fetch dependencies for Scala projects') + return subparsers + + +def main() -> None: + parser = argparse.ArgumentParser(add_help=False) + # parse_known_args does not support sub-commands so we split parsing into mutliple phases + build_base_parser(parser) + + args, unknown = parser.parse_known_args() + + if args.cwd: + os.chdir(args.cwd) + + if args.prepend_repo_path and args.repo_path: + args.repo_path = " ".join([args.prepend_repo_path, args.repo_path]) + elif args.prepend_repo_path: + args.repo_path = args.prepend_repo_path - args = parser.parse_args() if args.verbose == 4: log.setLevel('SPAM') elif args.verbose == 3: @@ -101,20 +121,23 @@ def main() -> None: log.debug("Log level: {}".format(log.getLevelName())) - if args.prepend_repo_path and args.repo_path: - args.repo_path = " ".join([args.prepend_repo_path, args.repo_path]) - elif args.prepend_repo_path: - args.repo_path = args.prepend_repo_path - if args.version: version() sys.exit(0) + # Now let's add the subparsers + parser = argparse.ArgumentParser(formatter_class=argparse.RawTextHelpFormatter) + build_base_parser(parser) + subparsers = add_sub_parsers(parser) + try: - # FIXME: This big switch statement... no good. - if args.command == 'init': - create(args) + # If the sub-command is init, then it's not a plugin command + if 'init' in unknown: + args = parser.parse_args() + assert args.command == 'init' + create(args) + # FIXME: This big switch statement... no good. else: # These commands assume the workspace already exists. Error out if the # workspace cannot be found. @@ -125,6 +148,18 @@ def main() -> None: log.error("Unable to find workspace root [{}]. Cannot continue.".format(e)) sys.exit(1) + ws.load_plugins() + + # Load plugins into parser + plugin_cmds = {} + for plugin in ws.plugins: + cmd = plugin.add_subparser(subparsers) + if cmd is not None: + plugin_cmds[cmd] = plugin + + args = parser.parse_args() + + # Built-in commands if args.command == 'add-pkg': add_pkg(ws, args) @@ -143,9 +178,6 @@ def main() -> None: elif args.command == 'update': update(ws, args) - elif args.command == 'fetch-scala': - fetch_scala(ws, args, agg=False) - elif args.command == 'inspect': if args.dot or args.tree: inspect_tree(ws, args) @@ -153,6 +185,17 @@ def main() -> None: log.error('`wit inspect` must be run with a flag') print(parser.parse_args('inspect -h'.split())) sys.exit(1) + + # Plugin commands + elif args.command in plugin_cmds: + plugin = plugin_cmds[args.command] + plugin.post_parse(ws, args, log) + + elif args.command == 'fetch-scala': + log.error("To install the scala plugin, run:\n" + " wit add-pkg https://github.com/sifive/wit-scala-plugin::" + "9246f3400b8fab6eccc828981026c9947d4a1b0c\n" + " wit update") except WitUserError as e: error(e) except AssertionError as e: @@ -187,9 +230,6 @@ def create(args) -> None: if args.update: update(ws, args) - if args.fetch_scala: - fetch_scala(ws, args, agg=True) - def add_pkg(ws, args) -> None: log.info("Adding package to workspace") @@ -364,52 +404,11 @@ def update(ws, args) -> None: print_errors(errors) sys.exit(1) + # Reload plugins after an update + ws.load_plugins() -def fetch_scala(ws, args, agg=True) -> None: - """Fetches bloop, coursier, and ivy dependencies - - It only fetches if ivydependencies.json files are found in packages - ws -- the Workspace - args -- arguments to the parser - agg -- indicates if this invocation is part of a larger command (like init) - """ - - # Collect ivydependency files - files = [] - for package in ws.lock.packages: - package.load(ws.root, False) - ivyfile = scalaplugin.ivy_deps_file(package.repo.path) - if os.path.isfile(ivyfile): - files.append(ivyfile) - else: - log.debug("No ivydependencies.json file found in package {}".format(package.name)) - - if len(files) == 0: - msg = "No ivydependencies.json files found, skipping fetching Scala..." - if agg: - log.debug(msg) - else: - # We want to print something if you run `wit fetch-scala` directly and nothing happens - log.info(msg) - else: - log.info("Fetching Scala install and dependencies...") - - install_dir = scalaplugin.scala_install_dir(ws.root) - - ivy_cache_dir = scalaplugin.ivy_cache_dir(ws.root) - os.makedirs(ivy_cache_dir, exist_ok=True) - - # Check if we need to install Bloop - if os.path.isdir(install_dir): - log.info("Scala install directory {} exists, skipping installation..." - .format(install_dir)) - else: - log.info("Installing Scala to {}...".format(install_dir)) - os.makedirs(install_dir, exist_ok=True) - scalaplugin.install_coursier(install_dir) - - log.info("Fetching ivy dependencies...") - scalaplugin.fetch_ivy_dependencies(files, install_dir, ivy_cache_dir) + for plugin in ws.plugins: + plugin.post_update(ws, args, log) def version() -> None: diff --git a/lib/wit/scala-bridge-fetcher_2.12-0.1.0.jar b/lib/wit/scala-bridge-fetcher_2.12-0.1.0.jar deleted file mode 100644 index 5a06788..0000000 Binary files a/lib/wit/scala-bridge-fetcher_2.12-0.1.0.jar and /dev/null differ diff --git a/lib/wit/scalaplugin.py b/lib/wit/scalaplugin.py deleted file mode 100755 index c59a4fd..0000000 --- a/lib/wit/scalaplugin.py +++ /dev/null @@ -1,221 +0,0 @@ -#!/usr/bin/env python3 - -import json -from collections import OrderedDict -import subprocess -import os -import urllib.request -from .witlogger import getLogger -from typing import List, Tuple - -log = getLogger() - - -SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) -SCRIPT_NAME = os.path.basename(__file__) - - -def scala_install_dir(path): - return str(path / "scala") - - -def ivy_cache_dir(path): - return str(path / "ivycache") - - -def coursier_bin(install_dir): - return "{}/coursier".format(install_dir) - - -def bloop_home(install_dir): - """ - The directory bloop thinks is $HOME - """ - return "{}/bloop_home".format(install_dir) - - -def ivy_deps_file(directory): - return "{}/ivydependencies.json".format(directory) - - -def scala_version_dep(version): - return "org.scala-lang:scala-compiler:{}".format(version) - - -def get_bloop_artifacts(): - version = "2.12.8" - # The version of bsp4s that bloop depeneds on isn't published so we override it - deps = ["ch.epfl.scala::bloop-frontend:1.2.5", "ch.epfl.scala::bsp4s:2.0.0-M3"] - expanded = [expand_scala_dep(version, dep) for dep in deps] - allDeps = expanded + [scala_version_dep(version)] - return allDeps - - -def bloop_classpath(coursier, cache, offline=True): - deps = get_bloop_artifacts() - offlineArgs = ["-m", "offline"] if offline else [] - cmd = [coursier, "fetch"] + offlineArgs + ["--classpath", "--cache", cache] + deps - proc = subprocess.run(cmd, stdout=subprocess.PIPE, universal_newlines=True) - return proc.stdout.rstrip() - - -def run_bloop(coursier, bloop_home, cache, args): - classpath = bloop_classpath(coursier, cache) - if classpath is None: - return 1 - arglist = args.split() - set_home = "-Duser.home={}".format(bloop_home) - cmd = ["java", "-Xss8M", set_home, "-cp", classpath, "bloop.Cli"] + arglist - proc = subprocess.run(cmd) - return proc.returncode == 0 - - -def fetch_scala_compiler_bridge(coursier, bloop_home, cache, version): - classpath = bloop_classpath(coursier, cache, offline=False) - if classpath is None: - return 1 - fetcher = "{}/scala-bridge-fetcher_2.12-0.1.0.jar".format(SCRIPT_DIR) - classpath = classpath + ":" + fetcher - # Make sure bloop_home exists - os.makedirs(bloop_home, mode=0o755, exist_ok=True) - set_home = "-Duser.home={}".format(bloop_home) - cmd = ["java", set_home, "-cp", classpath, "sifive.ScalaCompilerBridgeFetcher", version] - # This creates a target directory, put it in bloop_home - proc = subprocess.Popen(cmd, cwd=bloop_home) - proc.wait() - return proc.returncode == 0 - - -def install_coursier(install_dir): - version = "1.1.0-M14-6" - path = "io/get-coursier/coursier-cli_2.12/{}/coursier-cli_2.12-{}-standalone.jar".format( - version, version) - filename = coursier_bin(install_dir) - url = "http://central.maven.org/maven2/{}".format(path) - print("Downloading from {}".format(url)) - urllib.request.urlretrieve(url, filename) - os.chmod(filename, 0o755) - - -def split_scala_version(version): - parts = version.split('.') - if len(parts) != 3: - raise Exception("Malformed Scala Version {}".format(version)) - if parts[0] != "2": - raise Exception("Only Scala 2.X.Y are supported!") - return parts - - -def get_major_version(version): - return '.'.join(split_scala_version(version)[:2]) - - -def unique_list(l): - d = OrderedDict() - for e in l: - d[e] = None - return list(d.keys()) - - -# TODO More validation? -def expand_scala_dep(version, dep): - parts = dep.split(':') - - def errMalformed(): - raise Exception("Malformed IvyDependency {}!".format(dep)) - - def assertHasScala(): - if version is None: - raise Exception("Must specify scalaVersion for IvyDependency {}!".format(dep)) - - if len(parts) == 3: - # Java dep - return dep - elif len(parts) == 4: - # Scala Dep - c = parts.pop(1) - if c != '': - errMalformed() - assertHasScala() - sv = split_scala_version(version) - parts[1] = "{}_{}.{}".format(parts[1], sv[0], sv[1]) - return ':'.join(parts) - elif len(parts) == 5: - c = parts.pop(1) - d = parts.pop(1) - if c != '' or d != '': - errMalformed() - assertHasScala() - parts[1] = "{}_{}".format(parts[1], version) - return ':'.join(parts) - else: - errMalformed() - - -# TODO JSON validation/schema? -def read_ivy_file(filename): - with open(filename, 'r') as json_file: - data = json.load(json_file, object_pairs_hook=OrderedDict) - # Ignore project names, could be duplicates? - return list(data.values()) - return [] - - -def filter_versions(allVers, myVers): - """ - Determines what versions should be kept out of myVers based on major Scala version - """ - majorVersions = set([get_major_version(ver) for ver in allVers]) - return [ver for ver in myVers if get_major_version(ver) in majorVersions] - - -def resolve_dependencies(projects: List[dict]) -> Tuple[List[tuple], List[str]]: - """ - Determines which dependencies should be fetched - crossScalaVersions are used to fetch extra versions if any project has a - scalaVersion that matches the *major* version of the crossScalaVersion - """ - scalaVersions = unique_list(filter(None, [proj.get('scalaVersion') for proj in projects])) - dep_groups = [] - scala_versions = [] - for proj in projects: - version = proj.get('scalaVersion') - if version is not None: - scala_versions.append(version) - pdeps = proj.get('dependencies') or [] - crossVersions = proj.get('crossScalaVersions') or [] - # Note version can be none, this is okay - allVersions = [version] + filter_versions(scalaVersions, crossVersions) - for ver in allVersions: - deps = [expand_scala_dep(ver, dep) for dep in pdeps] - if ver is not None: - deps.append("org.scala-lang:scala-compiler:{}".format(ver)) - dep_groups.append(tuple(deps)) - unique_groups = unique_list(dep_groups) - unique_versions = unique_list(scala_versions) - return (unique_groups, unique_versions) - - -def fetch_ivy_deps(coursier: str, cache: str, deps: tuple) -> None: - log.debug("Fetching [{}]...".format(", ".join(deps))) - cmd = [coursier, "fetch", "--cache", cache] + list(deps) - proc = subprocess.run(cmd) - if proc.returncode != 0: - raise Exception("Unable to fetch dependencies [{}]".format(", ".join(deps))) - - -def fetch_ivy_dependencies(dep_files, install_dir, ivy_cache_dir): - coursier = coursier_bin(install_dir) - - projects = [] - for fn in dep_files: - projects.extend(read_ivy_file(fn)) - - (dep_groups, scala_versions) = resolve_dependencies(projects) - - bhome = bloop_home(install_dir) - for ver in scala_versions: - assert fetch_scala_compiler_bridge(coursier, bhome, ivy_cache_dir, ver) - - for group in dep_groups: - fetch_ivy_deps(coursier, ivy_cache_dir, group) diff --git a/lib/wit/workspace.py b/lib/wit/workspace.py index 04dd78c..22fdf1d 100644 --- a/lib/wit/workspace.py +++ b/lib/wit/workspace.py @@ -4,6 +4,8 @@ import shutil from pathlib import Path from pprint import pformat +import importlib.util +import inspect from .manifest import Manifest from .dependency import Dependency, sources_conflict_check from .lock import LockFile @@ -65,6 +67,7 @@ def __init__(self, root, repo_paths): self.repo_paths = repo_paths self.manifest = self._load_manifest() self.lock = self._load_lockfile() + self.plugins = [] def tag(self): return "[root]" @@ -178,6 +181,26 @@ def resolve_deps(self, wsroot, repo_paths, download, source_map, packages, queue return source_map, packages, queue, [] + def load_plugins(self): + for package in self.lock.packages: + package.load(self.root, False) + if package.repo is None: + log.debug("Cannot find source for package '{}'".format(package.name)) + continue + path = package.repo.path + plugin_path = "{}/wit-plugin.py".format(path) + if Path(plugin_path).is_file(): + log.debug("Found plugin file at [{}]".format(plugin_path)) + # Load the file + spec = importlib.util.spec_from_file_location("wit-plugin", plugin_path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + for name, value in inspect.getmembers(module): + if inspect.isclass(value): + if name == 'WitInterface': + log.debug("Found plugin '{}' in '{}'".format(name, plugin_path)) + self.plugins.append(value()) + def checkout(self, packages): lock_packages = [] for name in packages: diff --git a/t/regress.sh b/t/regress.sh index 5a78638..2bccd12 100755 --- a/t/regress.sh +++ b/t/regress.sh @@ -42,7 +42,7 @@ for test_path in $test_root/*.t; do then continue fi - echo -n "Running test [$test_name]" + # echo -n "Running test [$test_name]" mkdir $test_name cd $test_name @@ -58,7 +58,7 @@ for test_path in $test_root/*.t; do regression_result=1 fi - echo -e "\033[1K\033[1G${test_results[$test_name]} - ${test_name}"; + echo "${test_results[$test_name]} - ${test_name}"; done echo diff --git a/t/scala_plugin.t b/t/scala_plugin.t index 6c4d5a7..8f31d07 100755 --- a/t/scala_plugin.t +++ b/t/scala_plugin.t @@ -19,6 +19,16 @@ cat << EOF | jq . > foo/ivydependencies.json } EOF +cat << EOF | jq . > foo/wit-manifest.json +[ + { + "commit": "9246f3400b8fab6eccc828981026c9947d4a1b0c", + "name": "wit-scala-plugin", + "source": "https://github.com/sifive/wit-scala-plugin" + } +] +EOF + git -C foo add -A git -C foo commit -m "add ivydependencies.json" diff --git a/t/scala_plugin_cross_deps.t b/t/scala_plugin_cross_deps.t index ea5d9a3..5d31c52 100755 --- a/t/scala_plugin_cross_deps.t +++ b/t/scala_plugin_cross_deps.t @@ -19,6 +19,16 @@ cat << EOF | jq . > foo/ivydependencies.json } EOF +cat << EOF | jq . > foo/wit-manifest.json +[ + { + "commit": "9246f3400b8fab6eccc828981026c9947d4a1b0c", + "name": "wit-scala-plugin", + "source": "https://github.com/sifive/wit-scala-plugin" + } +] +EOF + git -C foo add -A git -C foo commit -m "add ivydependencies.json" diff --git a/t/scala_plugin_missing_scala_version.t b/t/scala_plugin_missing_scala_version.t index 33ad3e2..5d80267 100755 --- a/t/scala_plugin_missing_scala_version.t +++ b/t/scala_plugin_missing_scala_version.t @@ -17,6 +17,16 @@ cat << EOF | jq . > foo/ivydependencies.json } EOF +cat << EOF | jq . > foo/wit-manifest.json +[ + { + "commit": "9246f3400b8fab6eccc828981026c9947d4a1b0c", + "name": "wit-scala-plugin", + "source": "https://github.com/sifive/wit-scala-plugin" + } +] +EOF + git -C foo add -A git -C foo commit -m "add ivydependencies.json" diff --git a/t/scala_plugin_no_cross.t b/t/scala_plugin_no_cross.t index 4ffd746..6f553b4 100755 --- a/t/scala_plugin_no_cross.t +++ b/t/scala_plugin_no_cross.t @@ -18,6 +18,16 @@ cat << EOF | jq . > foo/ivydependencies.json } EOF +cat << EOF | jq . > foo/wit-manifest.json +[ + { + "commit": "9246f3400b8fab6eccc828981026c9947d4a1b0c", + "name": "wit-scala-plugin", + "source": "https://github.com/sifive/wit-scala-plugin" + } +] +EOF + git -C foo add -A git -C foo commit -m "add ivydependencies.json" diff --git a/t/scala_plugin_no_ivydependencies.t b/t/scala_plugin_no_ivydependencies.t index 5e15469..7357702 100755 --- a/t/scala_plugin_no_ivydependencies.t +++ b/t/scala_plugin_no_ivydependencies.t @@ -13,6 +13,9 @@ prereq "off" wit init myws -a $PWD/foo cd myws +wit add-pkg https://github.com/sifive/wit-scala-plugin::9246f3400b8fab6eccc828981026c9947d4a1b0c +wit update + wit fetch-scala check "wit fetch-scala should succeed" [ $? -eq 0 ] diff --git a/t/scala_plugin_no_scala.t b/t/scala_plugin_no_scala.t index 082f017..4d17911 100755 --- a/t/scala_plugin_no_scala.t +++ b/t/scala_plugin_no_scala.t @@ -17,6 +17,16 @@ cat << EOF | jq . > foo/ivydependencies.json } EOF +cat << EOF | jq . > foo/wit-manifest.json +[ + { + "commit": "9246f3400b8fab6eccc828981026c9947d4a1b0c", + "name": "wit-scala-plugin", + "source": "https://github.com/sifive/wit-scala-plugin" + } +] +EOF + git -C foo add -A git -C foo commit -m "add ivydependencies.json" diff --git a/t/scala_plugin_scalac_plugin.t b/t/scala_plugin_scalac_plugin.t index b57dc29..384a84c 100755 --- a/t/scala_plugin_scalac_plugin.t +++ b/t/scala_plugin_scalac_plugin.t @@ -18,6 +18,16 @@ cat << EOF | jq . > foo/ivydependencies.json } EOF +cat << EOF | jq . > foo/wit-manifest.json +[ + { + "commit": "9246f3400b8fab6eccc828981026c9947d4a1b0c", + "name": "wit-scala-plugin", + "source": "https://github.com/sifive/wit-scala-plugin" + } +] +EOF + git -C foo add -A git -C foo commit -m "add ivydependencies.json" diff --git a/t/wit_init_no_fetch_scala.t b/t/wit_init_no_fetch_scala.t index 05ebab8..7642b9a 100755 --- a/t/wit_init_no_fetch_scala.t +++ b/t/wit_init_no_fetch_scala.t @@ -18,6 +18,16 @@ cat << EOF | jq . > foo/ivydependencies.json } EOF +cat << EOF | jq . > foo/wit-manifest.json +[ + { + "commit": "9246f3400b8fab6eccc828981026c9947d4a1b0c", + "name": "wit-scala-plugin", + "source": "https://github.com/sifive/wit-scala-plugin" + } +] +EOF + git -C foo add -A git -C foo commit -m "add ivydependencies.json"