From 1f9332fb6dae8ec9fad10b752f7d54a8503817d0 Mon Sep 17 00:00:00 2001 From: Codrut Stancu Date: Thu, 25 Jul 2024 18:29:25 +0200 Subject: [PATCH] Implement Native Image Layers options. --- .../pointsto/heap/ImageLayerSnapshotUtil.java | 4 + .../com/oracle/svm/core/SubstrateOptions.java | 41 +--- .../svm/core/imagelayer/NativeImageLayers.md | 170 ++++++++++++++ .../oracle/svm/core/util/ArchiveSupport.java | 220 ++++++++++++++++++ .../com/oracle/svm/driver/BundleOptions.java | 69 ++++++ .../com/oracle/svm/driver/BundleSupport.java | 180 ++++---------- .../com/oracle/svm/driver/MacroOption.java | 3 +- .../com/oracle/svm/driver/NativeImage.java | 62 +---- .../hosted/NativeImageClassLoaderSupport.java | 10 +- .../svm/hosted/NativeImageGenerator.java | 37 +-- .../hosted/NativeImageGeneratorRunner.java | 8 +- .../HostedImageLayerBuildingSupport.java | 157 ++++++++++++- .../imagelayer/LayerArchiveSupport.java | 168 +++++++++++++ .../imagelayer/LayerOptionsSupport.java | 65 ++++++ .../imagelayer/LoadLayerArchiveSupport.java | 73 ++++++ .../imagelayer/WriteLayerArchiveSupport.java | 93 ++++++++ .../src/com/oracle/svm/util/LogUtils.java | 9 +- 17 files changed, 1110 insertions(+), 259 deletions(-) create mode 100644 substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/imagelayer/NativeImageLayers.md create mode 100644 substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/util/ArchiveSupport.java create mode 100644 substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleOptions.java create mode 100644 substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/imagelayer/LayerArchiveSupport.java create mode 100644 substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/imagelayer/LayerOptionsSupport.java create mode 100644 substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/imagelayer/LoadLayerArchiveSupport.java create mode 100644 substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/imagelayer/WriteLayerArchiveSupport.java diff --git a/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/heap/ImageLayerSnapshotUtil.java b/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/heap/ImageLayerSnapshotUtil.java index 4d09eb210c32..e6d3606388c7 100644 --- a/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/heap/ImageLayerSnapshotUtil.java +++ b/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/heap/ImageLayerSnapshotUtil.java @@ -88,6 +88,10 @@ public class ImageLayerSnapshotUtil { public static final String IMAGE_SINGLETON_KEYS = "image singleton keys"; public static final String IMAGE_SINGLETON_OBJECTS = "image singleton objects"; + public static String snapshotFileName(String imageName) { + return FILE_NAME_PREFIX + imageName + FILE_EXTENSION; + } + public String getTypeIdentifier(AnalysisType type) { String javaName = type.toJavaName(true); return addModuleName(javaName, type.getJavaClass().getModule().getName()); diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java index 35d6ea4eaaed..0c1fbb107bc4 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java @@ -107,21 +107,6 @@ public class SubstrateOptions { @Option(help = "Build shared library")// public static final HostedOptionKey SharedLibrary = new HostedOptionKey<>(false); - @Option(help = "Build a Native Image layer.")// - public static final HostedOptionKey ImageLayer = new HostedOptionKey<>(false) { - @Override - protected void onValueUpdate(EconomicMap, Object> values, Boolean oldValue, Boolean newValue) { - LayeredBaseImageAnalysis.update(values, newValue); - ClosedTypeWorld.update(values, !newValue); - StripDebugInfo.update(values, !newValue); - AOTTrivialInline.update(values, !newValue); - if (imageLayerEnabledHandler != null) { - imageLayerEnabledHandler.onOptionEnabled(values); - } - UseContainerSupport.update(values, !newValue); - } - }; - @APIOption(name = "static")// @Option(help = "Build statically linked executable (requires static libc and zlib)")// public static final HostedOptionKey StaticExecutable = new HostedOptionKey<>(false, key -> { @@ -134,6 +119,14 @@ protected void onValueUpdate(EconomicMap, Object> values, Boolean o } }); + // @APIOption(name = "layer-create")// + @Option(help = "Experimental: Build a Native Image layer.")// + public static final HostedOptionKey LayerCreate = new HostedOptionKey<>(AccumulatingLocatableMultiOptionValue.Strings.build()); + + // @APIOption(name = "layer-use")// + @Option(help = "Experimental: Build an image based on a Native Image layer.")// + public static final HostedOptionKey LayerUse = new HostedOptionKey<>(AccumulatingLocatableMultiOptionValue.Strings.build()); + @APIOption(name = "libc")// @Option(help = "Selects the libc implementation to use. Available implementations: glibc, musl, bionic")// public static final HostedOptionKey UseLibC = new HostedOptionKey<>(null) { @@ -195,7 +188,7 @@ protected void onValueUpdate(EconomicMap, Object> values, String ol public static final String IMAGE_MODULEPATH_PREFIX = "-imagemp"; public static final String KEEP_ALIVE_PREFIX = "-keepalive"; private static ValueUpdateHandler optimizeValueUpdateHandler; - private static OptionEnabledHandler imageLayerEnabledHandler; + public static OptionEnabledHandler imageLayerEnabledHandler; @Fold public static boolean getSourceLevelDebug() { @@ -1260,22 +1253,6 @@ public static boolean closedTypeWorld() { @Option(help = "Enables logging of failed hash code injection", type = OptionType.Debug) // public static final HostedOptionKey LoggingHashCodeInjection = new HostedOptionKey<>(false); - @Option(help = "Names of layer snapshots produced by PersistImageLayer", type = OptionType.Debug) // - @BundleMember(role = BundleMember.Role.Input)// - public static final HostedOptionKey LoadImageLayer = new HostedOptionKey<>(AccumulatingLocatableMultiOptionValue.Paths.build()) { - @Override - public void update(EconomicMap, Object> values, Object boxedValue) { - super.update(values, boxedValue); - ClosedTypeWorld.update(values, false); - /* Ignore any potential undefined references caused by inlining in base layer. */ - IgnoreUndefinedReferences.update(values, true); - AOTTrivialInline.update(values, false); - if (imageLayerEnabledHandler != null) { - imageLayerEnabledHandler.onOptionEnabled(values); - } - } - }; - public static class TruffleStableOptions { @Option(help = "Automatically copy the necessary language resources to the resources/languages directory next to the produced image." + diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/imagelayer/NativeImageLayers.md b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/imagelayer/NativeImageLayers.md new file mode 100644 index 000000000000..4e736db551eb --- /dev/null +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/imagelayer/NativeImageLayers.md @@ -0,0 +1,170 @@ +--- +layout: docs +toc_group: build-overview +link_title: Native Image Layers +permalink: /reference-manual/native-image/overview/NativeImageLayers/ +--- + +# Native Image Layers + +The Native Image Layers feature allows splitting an application and its required libraries into separate binaries: a +thin layer executable supported by a chain of shared libraries. +In contrast to regular Native Image building, this mode of operation enables sharing common libraries and VM/JDK code +between multiple applications, resulting in reduced memory usage at run time. +Moreover, it reduces the time to build an application since shared layers are only built once. + +> Note this feature is experimental and under development. + +### Table of Contents + +* [Native Image Layers Architecture](#native-image-layers-architecture) +* [Creating Native Image Layers](#creating-native-image-layers) +* [Packaging Native Image Layers](#packaging-native-image-layers) + +## Native Image Layers Architecture + +When using Native Image Layers an application is logically composed of the final application executable plus one or more +supporting layers containing code that the application requires. +The supporting layers are called _shared layers_ and the application executable is referred to as either the +_application layer_ or _executable layer_. +The _initial_ or _base_ layer is a shared layer containing VM internals and core _java.base_ functionality at a minimum. +It can also contain modules specific to a certain framework that the application may be built upon. +We refer to any subsequent layer built on top of a shared layer as an _extension layer_. + +At run time a shared layer is a shared object file on which other intermediate layers or executable application +layers can be dependent. +Thus, the application and its supporting shared layers form a chain: + +```shell +base-layer.so # initial layer (includes VM/JDK code) +└── mid-layer.so # intermediate layer, depends on base-layer.so, adds extra functionality + └── executable-image # final application executable, depends on mid-layer.so and base-layer.so +``` + +This architecture enables the sharing of layers between applications when the hierarchy of layers forms a tree structure. +For example, at run time there could be four applications that share one base layer and two intermediate layers: + +```shell +base-layer.so # initial layer (includes VM/JDK code) +├── executable-image-0 # final application executable, depends on base-layer.so +├── mid-layer-0.so # intermediate layer, depends on base-layer.so, adds extra functionality +│ ├── executable-image-00 # final application executable, depends on mid-layer-0.so and base-layer.so +│ └── executable-image-01 # final application executable, depends on mid-layer-0.so and base-layer.so +└── mid-layer-1.so # intermediate layer, depends on base-layer.so, adds extra functionality + └── executable-image-10 # final application executable, depends on mid-layer-1.so and base-layer.so +``` + +> Note: The current implementation is limited to only a base layer and an application layer. + +## Creating Native Image Layers + +> Note: The API options described in this section are experimental and not yet released. +> To interact with layers you must currently use their hosted variant: -H:LayerCreate and -H:LayerUse. + +To create and use layers `native-image` accepts two options: `--layer-create` and `--layer-use`. + +First, `--layer-create` builds an image layer archive from code available on the class or module path: + +``` +--layer-create=[layer-file.nil][,module=][,package=] + builds an image layer file from the modules and packages specified by "module" and "package". + The file name, if specified, must be a simple file name, i.e., not contain any path separators, + and have the *.nil extension. Otherwise, the layer-file name is derived from the image name. + This will generate a Native Image Layer archive file containing metadata required to build + either another layer or a final application executable that depends on this layer. + The archive also contains the shared object file corresponding to this layer. + If this option is specified with an empty value then it disables any prior layer creation option on the command line. +``` + +A layer archive file has a _.nil_ extension, acronym for **N**ative **I**mage **L**ayer. + +Second, `--layer-use` consumes a shared layer, and can extend it or create a final executable: + +``` +--layer-use=layer-file.nil + loads the given layer archive and makes it available to the build process. + If option --layer-create=another-layer.nil is specified this creates a new layer that depends on the loaded layer. + If no other layer option is specified this creates a final application executable that depends on the loaded layer. + If this option is specified with an empty value then it disables any prior layer application option on the command line. +``` + +Specifying each option more than once is allowed, however all but the last instance is ignored. +Passing an empty value will disable the option, overwriting any previous value. +These capabilities are useful in builds with long dependency chains where one may want to overwrite or disable layer +related options from a library dependency. + +#### Example Layers Option Usage + +```shell +# given an application that would usually be built like this +native-image --module-path target/AwesomeLib-1.0-SNAPSHOT.jar --another-extra-option -cp . AwesomeHelloWorld + +# you can now create a base-layer.nil layer from the java.base and java.awesome.lib modules +native-image --layer-create=base-layer.nil,module=java.base,module=java.awesome.lib --module-path target/AwesomeLib-1.0-SNAPSHOT.jar + +# then build the application on top of the preexisting base layer +native-image --layer-use=base-layer.nil --module-path target/AwesomeLib-1.0-SNAPSHOT.jar --another-extra-option -cp . AwesomeHelloWorld + +# extend the base layer with a mid layer that adds an extra module, chaining the layers together +native-image --layer-use=base-layer.nil --layer-create=mid-layer.nil,module=java.ultimate.io.lib --module-path target/UltimateIoLib-1.0-SNAPSHOT.jar + +# create an executable based on the mid layer (and implicitly on the base layer) and using the additional UltimateAwesomeHelloWorld.class from the classpath +native-image --layer-use=mid-layer.nil --module-path target/AwesomeLib-1.0-SNAPSHOT.jar:target/UltimateIoLib-1.0-SNAPSHOT.jar -cp . UltimateAwesomeHelloWorld + +# the same application can be built directly on the base-layer.nil +native-image --layer-use=base-layer.nil --module-path target/AwesomeLib-1.0-SNAPSHOT.jar:target/UltimateIoLib-1.0-SNAPSHOT.jar -cp . UltimateAwesomeHelloWorld + +# again, the same application can be built without any layers +native-image --module-path target/AwesomeLib-1.0-SNAPSHOT.jar:target/UltimateIoLib-1.0-SNAPSHOT.jar -cp . UltimateAwesomeHelloWorld + +# additionally a shared library can be built as a top layer containing the extra C entry points, based on a preexisting layer +native-image --layer-use=base-layer.nil --module-path target/AwesomeLib-1.0-SNAPSHOT.jar --shared + +# or without the layer +native-image --module-path target/AwesomeLib-1.0-SNAPSHOT.jar --shared + +``` + +### Invariants + +1. Every image build only creates one layer. + +2. The image build command used to create the application layer must work without `--layer-use` and create a standalone image. + The standalone command must completely specify the module/classpath and all other necessary configuration options + +3. The layer specified by `--layer-use` must be compatible with the standalone command line. + The compatibility rules refer to: + - class/jar file compatibility (`-cp`, `-p`, same JDK, same GraalVM, same libs, etc.) + - config compatibility: GC config, etc. + - Java system properties and Environment variables compatibility + - access compatibility: no additional unsafe and field accesses are allowed + + In case of incompatibility an error message is printed and the build process is aborted. + +### Limitations + +- Layers are platform dependent. A layer created with `--layer-create` on a specific OS/architecture + cannot be loaded by `--layer-use` on a different OS/architecture. +- Each layer can only depend on a previous layer. We explicitly make it impossible to depend on more than one layer to + avoid any potential issues that can stem from _multiple inheritance_. +- A shared layer is using the _.so_ extension to conform with the standard OS loader restrictions. However, it is not a + standard shared library file, and it cannot be used with other applications. + +## Packaging Native Image Layers + +At build time a shared layer is stored in a layer archive that contains the following artifacts: + +```shell +[shared-layer.nil] # shared layer archive file + ├── shared-layer.json # snapshot of the shared layer metadata; used by subsequent build processes + ├── shared-layer.so # shared object of the shared layer; used by subsequent build processes and at run time + └── shared-layer.properties # contains info about layer input data +``` + +The layer snapshot file will be consumed by subsequent build processes that depend on this layer. +It contains Native Image metadata, such as the analysis universe and available image singletons. +The shared object file will be used at build time for symbol resolution, and at run time for application execution. +The layer properties file contains metadata that uniquely identifies this layer: the options used to create the +layer, all the input files and their checksum. +Subsequent layer builds use the properties file to validate the layers they depend on: the JAR files that the build +depends on must exactly match those that were used to build the previous layers. diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/util/ArchiveSupport.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/util/ArchiveSupport.java new file mode 100644 index 000000000000..c402de7b7804 --- /dev/null +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/util/ArchiveSupport.java @@ -0,0 +1,220 @@ +/* + * Copyright (c) 2024, 2024, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package com.oracle.svm.core.util; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.ZonedDateTime; +import java.time.format.DateTimeFormatter; +import java.time.format.DateTimeParseException; +import java.time.format.FormatStyle; +import java.util.Collections; +import java.util.Comparator; +import java.util.Enumeration; +import java.util.HashMap; +import java.util.Map; +import java.util.Properties; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.Function; +import java.util.function.Predicate; +import java.util.jar.Attributes; +import java.util.jar.JarEntry; +import java.util.jar.JarFile; +import java.util.jar.JarOutputStream; +import java.util.jar.Manifest; +import java.util.stream.Stream; + +import com.oracle.svm.util.LogUtils; + +public class ArchiveSupport { + + final boolean isVerbose; + + public ArchiveSupport(boolean isVerbose) { + this.isVerbose = isVerbose; + } + + public void compressDirToJar(Path inputRootDir, Path outputFilePath, Manifest manifest) { + try (JarOutputStream jarOutStream = new JarOutputStream(Files.newOutputStream(outputFilePath), manifest)) { + try (Stream walk = Files.walk(inputRootDir)) { + walk.filter(Predicate.not(Files::isDirectory)).forEach(entry -> addFileToJar(inputRootDir, entry, outputFilePath, jarOutStream)); + } + } catch (IOException e) { + throw VMError.shouldNotReachHere("Failed to create JAR file " + outputFilePath.getFileName(), e); + } + } + + public void addFileToJar(Path inputDir, Path inputFile, Path outputFilePath, JarOutputStream jarOutStream) { + String jarEntryName = inputDir.relativize(inputFile).toString(); + JarEntry entry = new JarEntry(jarEntryName.replace(File.separator, "/")); + try { + entry.setTime(Files.getLastModifiedTime(inputFile).toMillis()); + jarOutStream.putNextEntry(entry); + Files.copy(inputFile, jarOutStream); + jarOutStream.closeEntry(); + } catch (IOException e) { + throw VMError.shouldNotReachHere("Failed to copy " + inputFile + " into JAR file " + outputFilePath.getFileName(), e); + } + } + + public Manifest createManifest() { + return createManifest(null); + } + + public Manifest createManifest(String mainClass) { + Manifest mf = new Manifest(); + Attributes attributes = mf.getMainAttributes(); + attributes.put(Attributes.Name.MANIFEST_VERSION, "1.0"); + if (mainClass != null) { + attributes.put(Attributes.Name.MAIN_CLASS, mainClass); + } + return mf; + } + + public void expandJarToDir(Path inputJarFilePath, Path outputDir, AtomicBoolean outputDirDeleted) { + expandJarToDir(Function.identity(), inputJarFilePath, outputDir, outputDirDeleted); + } + + public void expandJarToDir(Function relativizeEntry, Path inputJarFilePath, Path outputDir, AtomicBoolean outputDirDeleted) { + try { + try (JarFile archive = new JarFile(inputJarFilePath.toFile())) { + Enumeration jarEntries = archive.entries(); + while (jarEntries.hasMoreElements() && !outputDirDeleted.get()) { + JarEntry jarEntry = jarEntries.nextElement(); + Path originalEntry = outputDir.resolve(jarEntry.getName()); + Path targetEntry = relativizeEntry.apply(originalEntry); + try { + Path targetParent = targetEntry.getParent(); + if (targetParent != null) { + Files.createDirectories(targetParent); + } + Files.copy(archive.getInputStream(jarEntry), targetEntry); + } catch (IOException e) { + throw VMError.shouldNotReachHere("Unable to copy " + jarEntry.getName() + " from " + targetEntry + " to " + targetEntry, e); + } + } + } + } catch (IOException e) { + throw VMError.shouldNotReachHere("Unable to expand JAR file " + inputJarFilePath.getFileName(), e); + } + } + + public static Map loadProperties(Path propertiesPath) { + if (Files.isReadable(propertiesPath)) { + try { + return loadProperties(Files.newInputStream(propertiesPath)); + } catch (IOException e) { + throw VMError.shouldNotReachHere("Could not read properties-file: " + propertiesPath, e); + } + } + return Collections.emptyMap(); + } + + public static Map loadProperties(InputStream propertiesInputStream) { + Properties properties = new Properties(); + try (InputStream input = propertiesInputStream) { + properties.load(input); + } catch (IOException e) { + throw VMError.shouldNotReachHere("Could not read properties", e); + } + Map map = new HashMap<>(); + for (String key : properties.stringPropertyNames()) { + map.put(key, properties.getProperty(key)); + } + return Collections.unmodifiableMap(map); + } + + private static final String deletedFileSuffix = ".deleted"; + + private static boolean isDeletedPath(Path toDelete) { + Path fileName = toDelete.getFileName(); + if (fileName == null) { + throw VMError.shouldNotReachHere("Cannot determine file name for path."); + } + return fileName.toString().endsWith(deletedFileSuffix); + } + + public void deleteAllFiles(Path toDelete) { + try { + Path deletedPath = toDelete; + if (!isDeletedPath(deletedPath)) { + deletedPath = toDelete.resolveSibling(toDelete.getFileName() + deletedFileSuffix); + Files.move(toDelete, deletedPath); + } + Files.walk(deletedPath).sorted(Comparator.reverseOrder()).map(Path::toFile).forEach(File::delete); + } catch (IOException e) { + if (isVerbose) { + LogUtils.info("Could not recursively delete path: " + toDelete); + e.printStackTrace(); + } + } + } + + public Path createTempDir(String tempDirPrefix, AtomicBoolean tempDirDeleted) { + try { + Path tempDir = Files.createTempDirectory(tempDirPrefix); + Runtime.getRuntime().addShutdownHook(new Thread(() -> { + tempDirDeleted.set(true); + deleteAllFiles(tempDir); + })); + return tempDir; + } catch (IOException e) { + throw VMError.shouldNotReachHere("Unable to create temp directory for prefix " + tempDirPrefix, e); + } + } + + public void ensureDirectoryExists(Path dir) { + if (Files.exists(dir)) { + if (!Files.isDirectory(dir)) { + throw VMError.shouldNotReachHere("File " + dir + " is not a directory"); + } + } else { + try { + Files.createDirectories(dir); + } catch (IOException e) { + throw VMError.shouldNotReachHere("Could not create directory " + dir); + } + } + } + + public static String currentTime() { + return ZonedDateTime.now().format(DateTimeFormatter.ISO_DATE_TIME); + } + + public static String parseTimestamp(String timestamp) { + String localDateStr; + try { + ZonedDateTime dateTime = ZonedDateTime.parse(timestamp, DateTimeFormatter.ISO_DATE_TIME); + localDateStr = dateTime.format(DateTimeFormatter.ofLocalizedDateTime(FormatStyle.FULL)); + } catch (DateTimeParseException e) { + localDateStr = "unknown time"; + } + return localDateStr; + } + +} diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleOptions.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleOptions.java new file mode 100644 index 000000000000..537cebaf646e --- /dev/null +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleOptions.java @@ -0,0 +1,69 @@ +/* + * Copyright (c) 2024, 2024, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package com.oracle.svm.driver; + +import static com.oracle.svm.driver.BundleSupport.BUNDLE_OPTION; + +import java.util.Arrays; + +import com.oracle.svm.core.SubstrateUtil; + +public class BundleOptions { + + /** Split a bundle argument into its components. */ + public static BundleOption parseBundleOption(String cmdLineArg) { + // Given an argument of form --bundle-create=bundle.nib,dry-run + // First get the list: [bundle-create=bundle.nib, dry-run] + String[] options = SubstrateUtil.split(cmdLineArg.substring(BUNDLE_OPTION.length() + 1), ","); + // Then extract the variant components: [create=bundle.nib, dry-run] + String[] variantAndFileName = SubstrateUtil.split(options[0], "=", 2); + // First part is the option variant + String variant = variantAndFileName[0]; + // Second part is the optional file name + String fileName = null; + if (variantAndFileName.length == 2) { + fileName = variantAndFileName[1]; + } + // The rest are optional extended options + ExtendedOption[] extendedOptions = Arrays.stream(options).skip(1).map(BundleOptions::parseExtendedOption).toArray(ExtendedOption[]::new); + return new BundleOption(variant, fileName, extendedOptions); + } + + public record BundleOption(String variant, String fileName, ExtendedOption[] extendedOptions) { + } + + private static ExtendedOption parseExtendedOption(String option) { + String[] optionParts = SubstrateUtil.split(option, "=", 2); + if (optionParts.length == 2) { + return new ExtendedOption(optionParts[0], optionParts[1]); + } else { + return new ExtendedOption(option, null); + } + } + + public record ExtendedOption(String key, String value) { + } + +} diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java index 6da7f4bb78c5..4164b8d5382e 100644 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java @@ -24,7 +24,6 @@ */ package com.oracle.svm.driver; -import java.io.File; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; @@ -36,14 +35,9 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.nio.file.StandardCopyOption; -import java.time.ZonedDateTime; -import java.time.format.DateTimeFormatter; -import java.time.format.DateTimeParseException; -import java.time.format.FormatStyle; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; -import java.util.Enumeration; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -56,16 +50,15 @@ import java.util.UUID; import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.Predicate; -import java.util.jar.Attributes; -import java.util.jar.JarEntry; import java.util.jar.JarFile; -import java.util.jar.JarOutputStream; import java.util.jar.Manifest; import java.util.stream.Stream; import com.oracle.svm.core.OS; -import com.oracle.svm.core.SubstrateUtil; import com.oracle.svm.core.option.BundleMember; +import com.oracle.svm.core.util.ArchiveSupport; +import com.oracle.svm.driver.BundleOptions.BundleOption; +import com.oracle.svm.driver.BundleOptions.ExtendedOption; import com.oracle.svm.driver.launcher.BundleLauncher; import com.oracle.svm.driver.launcher.ContainerSupport; import com.oracle.svm.driver.launcher.configuration.BundleArgsParser; @@ -142,18 +135,11 @@ String optionName() { static BundleSupport create(NativeImage nativeImage, String bundleArg, NativeImage.ArgumentQueue args) { try { - String bundleFilename = null; - String[] options = SubstrateUtil.split(bundleArg.substring(BUNDLE_OPTION.length() + 1), ","); - - String[] variantParts = SubstrateUtil.split(options[0], "=", 2); - String variant = variantParts[0]; - if (variantParts.length == 2) { - bundleFilename = variantParts[1]; - } + BundleOption bundleOption = BundleOptions.parseBundleOption(bundleArg); String applyOptionName = BundleOptionVariants.apply.optionName(); String createOptionName = BundleOptionVariants.create.optionName(); BundleSupport bundleSupport; - switch (BundleOptionVariants.valueOf(variant)) { + switch (BundleOptionVariants.valueOf(bundleOption.variant())) { case apply: if (nativeImage.useBundle()) { if (nativeImage.bundleSupport.loadBundle) { @@ -163,10 +149,10 @@ static BundleSupport create(NativeImage nativeImage, String bundleArg, NativeIma throw NativeImage.showError(String.format("native-image option %s is not allowed to be used after option %s.", applyOptionName, createOptionName)); } } - if (bundleFilename == null) { + if (bundleOption.fileName() == null) { throw NativeImage.showError(String.format("native-image option %s requires a bundle file argument. E.g. %s=bundle-file.nib.", applyOptionName, applyOptionName)); } - bundleSupport = new BundleSupport(nativeImage, bundleFilename); + bundleSupport = new BundleSupport(nativeImage, bundleOption.fileName()); /* Inject the command line args from the loaded bundle in-place */ List buildArgs = bundleSupport.getNativeImageArgs(); for (int i = buildArgs.size() - 1; i >= 0; i--) { @@ -187,17 +173,14 @@ static BundleSupport create(NativeImage nativeImage, String bundleArg, NativeIma } else { bundleSupport = new BundleSupport(nativeImage); } - if (bundleFilename != null) { - bundleSupport.updateBundleLocation(Path.of(bundleFilename), true); + if (bundleOption.fileName() != null) { + bundleSupport.updateBundleLocation(Path.of(bundleOption.fileName()), true); } break; default: throw new IllegalArgumentException(); } - - Arrays.stream(options) - .skip(1) - .forEach(bundleSupport::parseExtendedOption); + Arrays.stream(bundleOption.extendedOptions()).forEach(bundleSupport::processExtendedOption); if (!bundleSupport.useContainer && bundleSupport.bundleProperties.requireContainerBuild()) { if (!OS.LINUX.isCurrent()) { @@ -240,48 +223,36 @@ void createDockerfile(Path dockerfile) { } } - private void parseExtendedOption(String option) { - String optionKey; - String optionValue; - - String[] optionParts = SubstrateUtil.split(option, "=", 2); - if (optionParts.length == 2) { - optionKey = optionParts[0]; - optionValue = optionParts[1]; - } else { - optionKey = option; - optionValue = null; - } - - switch (optionKey) { + private void processExtendedOption(ExtendedOption option) { + switch (option.key()) { case DRY_RUN_OPTION -> nativeImage.setDryRun(true); case CONTAINER_OPTION -> { if (containerSupport != null) { - throw NativeImage.showError(String.format("native-image bundle allows option %s to be specified only once.", optionKey)); + throw NativeImage.showError(String.format("native-image bundle allows option %s to be specified only once.", option.key())); } containerSupport = new ContainerSupport(stageDir, NativeImage::showError, LogUtils::warning, nativeImage::showMessage); useContainer = true; - if (optionValue != null) { - if (!ContainerSupport.SUPPORTED_TOOLS.contains(optionValue)) { - throw NativeImage.showError(String.format("Container Tool '%s' is not supported, please use one of the following tools: %s", optionValue, ContainerSupport.SUPPORTED_TOOLS)); + if (option.value() != null) { + if (!ContainerSupport.SUPPORTED_TOOLS.contains(option.value())) { + throw NativeImage.showError(String.format("Container Tool '%s' is not supported, please use one of the following tools: %s", option.value(), ContainerSupport.SUPPORTED_TOOLS)); } - containerSupport.tool = optionValue; + containerSupport.tool = option.value(); } } case DOCKERFILE_OPTION -> { if (containerSupport == null) { - throw NativeImage.showError(String.format("native-image bundle option %s is only allowed to be used after option %s.", optionKey, CONTAINER_OPTION)); + throw NativeImage.showError(String.format("native-image bundle option %s is only allowed to be used after option %s.", option.key(), CONTAINER_OPTION)); } - if (optionValue != null) { - containerSupport.dockerfile = Path.of(optionValue); + if (option.value() != null) { + containerSupport.dockerfile = Path.of(option.value()); if (!Files.isReadable(containerSupport.dockerfile)) { throw NativeImage.showError(String.format("Dockerfile '%s' is not readable", containerSupport.dockerfile.toAbsolutePath())); } } else { - throw NativeImage.showError(String.format("native-image option %s requires a dockerfile argument. E.g. %s=path/to/Dockerfile.", optionKey, optionKey)); + throw NativeImage.showError(String.format("native-image option %s requires a dockerfile argument. E.g. %s=path/to/Dockerfile.", option.key(), option.key())); } } - default -> throw NativeImage.showError(String.format("Unknown option %s. Use --help-extra for usage instructions.", optionKey)); + default -> throw NativeImage.showError(String.format("Unknown option %s. Use --help-extra for usage instructions.", option.key())); } } @@ -292,7 +263,7 @@ private BundleSupport(NativeImage nativeImage) { loadBundle = false; writeBundle = true; try { - rootDir = createBundleRootDir(); + rootDir = nativeImage.archiveSupport().createTempDir(BUNDLE_TEMP_DIR_PREFIX, deleteBundleRoot); bundleProperties = new BundleProperties(); bundleProperties.properties.put(BundleProperties.PROPERTY_KEY_IMAGE_BUILD_ID, UUID.randomUUID().toString()); @@ -322,38 +293,14 @@ private BundleSupport(NativeImage nativeImage, String bundleFilenameArg) { Objects.requireNonNull(bundleFilenameArg); updateBundleLocation(Path.of(bundleFilenameArg), false); - try { - rootDir = createBundleRootDir(); - bundleProperties = new BundleProperties(); - bundleProperties.properties.put(BundleProperties.PROPERTY_KEY_IMAGE_BUILD_ID, UUID.randomUUID().toString()); + rootDir = nativeImage.archiveSupport().createTempDir(BUNDLE_TEMP_DIR_PREFIX, deleteBundleRoot); + bundleProperties = new BundleProperties(); + bundleProperties.properties.put(BundleProperties.PROPERTY_KEY_IMAGE_BUILD_ID, UUID.randomUUID().toString()); - outputDir = rootDir.resolve("output"); - String originalOutputDirName = outputDir.getFileName().toString() + ORIGINAL_DIR_EXTENSION; - - Path bundleFilePath = bundlePath.resolve(bundleName + BUNDLE_FILE_EXTENSION); - try (JarFile archive = new JarFile(bundleFilePath.toFile())) { - Enumeration jarEntries = archive.entries(); - while (jarEntries.hasMoreElements() && !deleteBundleRoot.get()) { - JarEntry jarEntry = jarEntries.nextElement(); - Path bundleEntry = rootDir.resolve(jarEntry.getName()); - if (bundleEntry.startsWith(outputDir)) { - /* Extract original output to different path */ - bundleEntry = rootDir.resolve(originalOutputDirName).resolve(outputDir.relativize(bundleEntry)); - } - try { - Path bundleFileParent = bundleEntry.getParent(); - if (bundleFileParent != null) { - Files.createDirectories(bundleFileParent); - } - Files.copy(archive.getInputStream(jarEntry), bundleEntry); - } catch (IOException e) { - throw NativeImage.showError("Unable to copy " + jarEntry.getName() + " from bundle " + bundleEntry + " to " + bundleEntry, e); - } - } - } - } catch (IOException e) { - throw NativeImage.showError("Unable to expand bundle directory layout from bundle file " + bundleName + BUNDLE_FILE_EXTENSION, e); - } + outputDir = rootDir.resolve("output"); + + Path bundleFilePath = bundlePath.resolve(bundleName + BUNDLE_FILE_EXTENSION); + nativeImage.archiveSupport().expandJarToDir(e -> relativizeBundleEntry(getOriginalOutputDirName(), e), bundleFilePath, rootDir, deleteBundleRoot); if (deleteBundleRoot.get()) { /* Abort image build request without error message and exit with 0 */ @@ -408,17 +355,20 @@ private BundleSupport(NativeImage nativeImage, String bundleFilenameArg) { } } - private final AtomicBoolean deleteBundleRoot = new AtomicBoolean(); + private Path relativizeBundleEntry(String originalOutputDirName, Path bundleEntry) { + if (bundleEntry.startsWith(outputDir)) { + /* Extract original output to different path */ + return rootDir.resolve(originalOutputDirName).resolve(outputDir.relativize(bundleEntry)); + } + return bundleEntry; + } - private Path createBundleRootDir() throws IOException { - Path bundleRoot = Files.createTempDirectory(BUNDLE_TEMP_DIR_PREFIX); - Runtime.getRuntime().addShutdownHook(new Thread(() -> { - deleteBundleRoot.set(true); - nativeImage.deleteAllFiles(bundleRoot); - })); - return bundleRoot; + private String getOriginalOutputDirName() { + return outputDir.getFileName().toString() + ORIGINAL_DIR_EXTENSION; } + private final AtomicBoolean deleteBundleRoot = new AtomicBoolean(); + public List getNativeImageArgs() { return nativeImageArgs; } @@ -685,15 +635,14 @@ void updateBundleLocation(Path bundleFile, boolean redefine) { } private Path writeBundle() { - String originalOutputDirName = outputDir.getFileName().toString() + ORIGINAL_DIR_EXTENSION; - Path originalOutputDir = rootDir.resolve(originalOutputDirName); + Path originalOutputDir = rootDir.resolve(getOriginalOutputDirName()); if (Files.exists(originalOutputDir)) { - nativeImage.deleteAllFiles(originalOutputDir); + nativeImage.archiveSupport().deleteAllFiles(originalOutputDir); } Path metaInfDir = rootDir.resolve(JarFile.MANIFEST_NAME); if (Files.exists(metaInfDir)) { - nativeImage.deleteAllFiles(metaInfDir); + nativeImage.archiveSupport().deleteAllFiles(metaInfDir); } Path bundleLauncherFile = Paths.get("/").resolve(BundleLauncher.class.getName().replace(".", "/") + ".class"); @@ -827,36 +776,12 @@ private Path writeBundle() { bundleProperties.write(); Path bundleFilePath = bundlePath.resolve(bundleName + BUNDLE_FILE_EXTENSION); - try (JarOutputStream jarOutStream = new JarOutputStream(Files.newOutputStream(bundleFilePath), createManifest())) { - try (Stream walk = Files.walk(rootDir)) { - walk.filter(Predicate.not(Files::isDirectory)).forEach(bundleEntry -> { - String jarEntryName = rootDir.relativize(bundleEntry).toString(); - JarEntry entry = new JarEntry(jarEntryName.replace(File.separator, "/")); - try { - entry.setTime(Files.getLastModifiedTime(bundleEntry).toMillis()); - jarOutStream.putNextEntry(entry); - Files.copy(bundleEntry, jarOutStream); - jarOutStream.closeEntry(); - } catch (IOException e) { - throw NativeImage.showError("Failed to copy " + bundleEntry + " into bundle file " + bundleFilePath.getFileName(), e); - } - }); - } - } catch (IOException e) { - throw NativeImage.showError("Failed to create bundle file " + bundleFilePath.getFileName(), e); - } + Manifest manifest = nativeImage.archiveSupport().createManifest(BundleLauncher.class.getName()); + nativeImage.archiveSupport().compressDirToJar(rootDir, bundleFilePath, manifest); return bundleFilePath; } - private static Manifest createManifest() { - Manifest mf = new Manifest(); - Attributes attributes = mf.getMainAttributes(); - attributes.put(Attributes.Name.MANIFEST_VERSION, "1.0"); - attributes.put(Attributes.Name.MAIN_CLASS, BundleLauncher.class.getName()); - return mf; - } - private static final String substitutionMapSrcField = "src"; private static final String substitutionMapDstField = "dst"; @@ -916,7 +841,7 @@ private void loadAndVerify() { throw NativeImage.showError("The given bundle file " + bundleFileName + " does not contain a bundle properties file"); } - properties.putAll(NativeImage.loadProperties(bundlePropertiesFile)); + properties.putAll(ArchiveSupport.loadProperties(bundlePropertiesFile)); String fileVersionKey = PROPERTY_KEY_BUNDLE_FILE_VERSION_MAJOR; try { int major = Integer.parseInt(properties.getOrDefault(fileVersionKey, "-1")); @@ -943,16 +868,9 @@ private void loadAndVerify() { String bundlePlatform = properties.getOrDefault(PROPERTY_KEY_NATIVE_IMAGE_PLATFORM, "unknown"); String currentPlatform = bundlePlatform.equals(NativeImage.platform) ? "" : " != '" + NativeImage.platform + "'"; String bundleCreationTimestamp = properties.getOrDefault(PROPERTY_KEY_BUNDLE_FILE_CREATION_TIMESTAMP, ""); - String localDateStr; - try { - ZonedDateTime dateTime = ZonedDateTime.parse(bundleCreationTimestamp, DateTimeFormatter.ISO_DATE_TIME); - localDateStr = dateTime.format(DateTimeFormatter.ofLocalizedDateTime(FormatStyle.FULL)); - } catch (DateTimeParseException e) { - localDateStr = "unknown time"; - } nativeImage.showNewline(); nativeImage.showMessage("%sLoaded Bundle from %s", BUNDLE_INFO_MESSAGE_PREFIX, bundleFileName); - nativeImage.showMessage("%sBundle created at '%s'", BUNDLE_INFO_MESSAGE_PREFIX, localDateStr); + nativeImage.showMessage("%sBundle created at '%s'", BUNDLE_INFO_MESSAGE_PREFIX, ArchiveSupport.parseTimestamp(bundleCreationTimestamp)); nativeImage.showMessage("%sUsing version: '%s'%s (vendor '%s'%s) on platform: '%s'%s", BUNDLE_INFO_MESSAGE_PREFIX, bundleVersion, currentVersion, bundleVendor, currentVendor, @@ -972,7 +890,7 @@ private boolean requireContainerBuild() { private void write() { properties.put(PROPERTY_KEY_BUNDLE_FILE_VERSION_MAJOR, String.valueOf(BUNDLE_FILE_FORMAT_VERSION_MAJOR)); properties.put(PROPERTY_KEY_BUNDLE_FILE_VERSION_MINOR, String.valueOf(BUNDLE_FILE_FORMAT_VERSION_MINOR)); - properties.put(PROPERTY_KEY_BUNDLE_FILE_CREATION_TIMESTAMP, ZonedDateTime.now().format(DateTimeFormatter.ISO_DATE_TIME)); + properties.put(PROPERTY_KEY_BUNDLE_FILE_CREATION_TIMESTAMP, ArchiveSupport.currentTime()); properties.put(PROPERTY_KEY_BUILDER_ON_CLASSPATH, String.valueOf(forceBuilderOnClasspath)); boolean imageBuilt = !nativeImage.isDryRun(); properties.put(PROPERTY_KEY_IMAGE_BUILT, String.valueOf(imageBuilt)); diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/MacroOption.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/MacroOption.java index d1a23064eecc..d0ef525d7294 100644 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/MacroOption.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/MacroOption.java @@ -45,6 +45,7 @@ import com.oracle.svm.core.option.OptionUtils; import com.oracle.svm.driver.NativeImage.BuildConfiguration; import com.oracle.svm.driver.metainf.NativeImageMetaInfWalker; +import com.oracle.svm.core.util.ArchiveSupport; final class MacroOption { @@ -382,7 +383,7 @@ private MacroOption(Path optionDirectory) { this.kind = OptionUtils.MacroOptionKind.fromSubdir(optionDirectory.getParent().getFileName().toString()); this.optionName = optionDirectory.getFileName().toString(); this.optionDirectory = optionDirectory; - this.properties = NativeImage.loadProperties(optionDirectory.resolve(NativeImageMetaInfWalker.nativeImagePropertiesFilename)); + this.properties = ArchiveSupport.loadProperties(optionDirectory.resolve(NativeImageMetaInfWalker.nativeImagePropertiesFilename)); } @Override diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java index 8009a21bec12..ec4ee2906a95 100644 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java @@ -44,7 +44,6 @@ import java.util.Arrays; import java.util.Collection; import java.util.Collections; -import java.util.Comparator; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; @@ -55,7 +54,6 @@ import java.util.Locale; import java.util.Map; import java.util.Optional; -import java.util.Properties; import java.util.Set; import java.util.StringJoiner; import java.util.function.BiConsumer; @@ -97,6 +95,7 @@ import com.oracle.svm.driver.metainf.NativeImageMetaInfWalker; import com.oracle.svm.hosted.NativeImageGeneratorRunner; import com.oracle.svm.hosted.NativeImageSystemClassLoader; +import com.oracle.svm.core.util.ArchiveSupport; import com.oracle.svm.hosted.util.JDKArgsUtils; import com.oracle.svm.util.LogUtils; import com.oracle.svm.util.ModuleSupport; @@ -257,7 +256,7 @@ private static String oR(OptionKey option) { final String oHUseLibC = oH(SubstrateOptions.UseLibC); final String oHEnableStaticExecutable = oHEnabled(SubstrateOptions.StaticExecutable); final String oHEnableSharedLibraryFlagPrefix = oHEnabled + SubstrateOptions.SharedLibrary.getName(); - final String oHEnableImageLayerFlagPrefix = oHEnabled + SubstrateOptions.ImageLayer.getName(); + final String oHEnableImageLayerFlagPrefix = oH + SubstrateOptions.LayerCreate.getName(); final String oHColor = oH(SubstrateOptions.Color); final String oHEnableBuildOutputProgress = oHEnabledByDriver(SubstrateOptions.BuildOutputProgress); final String oHEnableBuildOutputLinks = oHEnabledByDriver(SubstrateOptions.BuildOutputLinks); @@ -307,6 +306,7 @@ private static String oR(OptionKey option) { private long imageBuilderPid = -1; BundleSupport bundleSupport; + private final ArchiveSupport archiveSupport; protected static class BuildConfiguration { /* @@ -644,7 +644,7 @@ public boolean processMetaInfResource(Path classpathEntry, Path resourceRoot, Pa boolean ignoreClasspathEntry = false; Map properties = null; if (isNativeImagePropertiesFile) { - properties = loadProperties(Files.newInputStream(resourcePath)); + properties = ArchiveSupport.loadProperties(Files.newInputStream(resourcePath)); if (config.modulePathBuild) { String forceOnModulePath = properties.get("ForceOnModulePath"); if (forceOnModulePath != null) { @@ -815,11 +815,12 @@ public boolean buildFallbackImage() { protected NativeImage(BuildConfiguration config) { this.config = config; this.metaInfProcessor = new DriverMetaInfProcessor(); + this.archiveSupport = new ArchiveSupport(isVerbose()); String configFile = System.getenv(CONFIG_FILE_ENV_VAR_KEY); if (configFile != null && !configFile.isEmpty()) { try { - userConfigProperties.putAll(loadProperties(canonicalize(Paths.get(configFile)))); + userConfigProperties.putAll(ArchiveSupport.loadProperties(canonicalize(Paths.get(configFile)))); } catch (NativeImageError | Exception e) { showError("Invalid environment variable " + CONFIG_FILE_ENV_VAR_KEY, e); } @@ -1769,6 +1770,10 @@ boolean useBundle() { return bundleSupport != null; } + public ArchiveSupport archiveSupport() { + return archiveSupport; + } + @Deprecated private static void deprecatedSanitizeJVMEnvironment(Map environment) { String[] jvmAffectingEnvironmentVariables = {"JAVA_COMPILER", "_JAVA_OPTIONS", "JAVA_TOOL_OPTIONS", "JDK_JAVA_OPTIONS", "CLASSPATH"}; @@ -2397,31 +2402,6 @@ private boolean configureBuildOutput() { return useColorfulOutput; } - static Map loadProperties(Path propertiesPath) { - if (Files.isReadable(propertiesPath)) { - try { - return loadProperties(Files.newInputStream(propertiesPath)); - } catch (IOException e) { - throw showError("Could not read properties-file: " + propertiesPath, e); - } - } - return Collections.emptyMap(); - } - - static Map loadProperties(InputStream propertiesInputStream) { - Properties properties = new Properties(); - try (InputStream input = propertiesInputStream) { - properties.load(input); - } catch (IOException e) { - showError("Could not read properties", e); - } - Map map = new HashMap<>(); - for (String key : properties.stringPropertyNames()) { - map.put(key, properties.getProperty(key)); - } - return Collections.unmodifiableMap(map); - } - static boolean forEachPropertyValue(String propertyValue, Consumer target, Function resolver) { return forEachPropertyValue(propertyValue, target, resolver, "\\s+"); } @@ -2474,28 +2454,6 @@ private static String safeSubstitution(String source, CharSequence target, CharS return source.replace(target, replacement); } - private static final String deletedFileSuffix = ".deleted"; - - protected static boolean isDeletedPath(Path toDelete) { - return toDelete.getFileName().toString().endsWith(deletedFileSuffix); - } - - protected void deleteAllFiles(Path toDelete) { - try { - Path deletedPath = toDelete; - if (!isDeletedPath(deletedPath)) { - deletedPath = toDelete.resolveSibling(toDelete.getFileName() + deletedFileSuffix); - Files.move(toDelete, deletedPath); - } - Files.walk(deletedPath).sorted(Comparator.reverseOrder()).map(Path::toFile).forEach(File::delete); - } catch (IOException e) { - if (isVerbose()) { - showMessage("Could not recursively delete path: " + toDelete); - e.printStackTrace(); - } - } - } - private record ExcludeConfig(Pattern jarPattern, Pattern resourcePattern) { } } diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/NativeImageClassLoaderSupport.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/NativeImageClassLoaderSupport.java index 434764e5c638..a7097e6f9bae 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/NativeImageClassLoaderSupport.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/NativeImageClassLoaderSupport.java @@ -83,8 +83,8 @@ import com.oracle.svm.core.NativeImageClassLoaderOptions; import com.oracle.svm.core.SubstrateOptions; -import com.oracle.svm.core.option.HostedOptionKey; import com.oracle.svm.core.option.AccumulatingLocatableMultiOptionValue; +import com.oracle.svm.core.option.HostedOptionKey; import com.oracle.svm.core.option.OptionOrigin; import com.oracle.svm.core.option.SubstrateOptionsParser; import com.oracle.svm.core.util.ClasspathUtils; @@ -92,6 +92,7 @@ import com.oracle.svm.core.util.UserError; import com.oracle.svm.core.util.VMError; import com.oracle.svm.hosted.annotation.SubstrateAnnotationExtractor; +import com.oracle.svm.hosted.imagelayer.HostedImageLayerBuildingSupport; import com.oracle.svm.hosted.option.HostedOptionParser; import com.oracle.svm.util.ClassUtil; import com.oracle.svm.util.LogUtils; @@ -268,6 +269,13 @@ private static void missingFromSetOfEntriesError(Object entry, Collection all public void setupHostedOptionParser(List arguments) { hostedOptionParser = new HostedOptionParser(getClassLoader(), arguments); remainingArguments = Collections.unmodifiableList((hostedOptionParser.parse())); + /* + * The image layer support needs to be configured early to correctly set the + * class-path/module-path options. Note that parsedHostedOptions is a copy-by-value of + * hostedOptionParser.getHostedValues(), so we want to affect the options map before it is + * copied. + */ + HostedImageLayerBuildingSupport.processLayerOptions(hostedOptionParser.getHostedValues()); parsedHostedOptions = new OptionValues(hostedOptionParser.getHostedValues()); } diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/NativeImageGenerator.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/NativeImageGenerator.java index f912cec9a186..2603473e95fd 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/NativeImageGenerator.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/NativeImageGenerator.java @@ -99,7 +99,6 @@ import com.oracle.graal.pointsto.heap.ImageHeap; import com.oracle.graal.pointsto.heap.ImageHeapScanner; import com.oracle.graal.pointsto.heap.ImageLayerLoader; -import com.oracle.graal.pointsto.heap.ImageLayerSnapshotUtil; import com.oracle.graal.pointsto.infrastructure.SubstitutionProcessor; import com.oracle.graal.pointsto.infrastructure.WrappedJavaMethod; import com.oracle.graal.pointsto.meta.AnalysisFactory; @@ -126,7 +125,6 @@ import com.oracle.graal.reachability.SimpleInMemoryMethodSummaryProvider; import com.oracle.svm.common.meta.MultiMethod; import com.oracle.svm.core.BuildArtifacts; -import com.oracle.svm.core.BuildArtifacts.ArtifactType; import com.oracle.svm.core.BuildPhaseProvider; import com.oracle.svm.core.ClassLoaderSupport; import com.oracle.svm.core.JavaMainWrapper.JavaMainSupport; @@ -546,12 +544,12 @@ public void run(Map entryPoints, try (TemporaryBuildDirectoryProviderImpl tempDirectoryProvider = new TemporaryBuildDirectoryProviderImpl()) { ImageSingletons.add(TemporaryBuildDirectoryProvider.class, tempDirectoryProvider); if (ImageLayerBuildingSupport.buildingSharedLayer()) { - setupImageLayerArtifact(imageName); + HostedImageLayerBuildingSupport.setupImageLayerArtifact(imageName); } doRun(entryPoints, javaMainSupport, imageName, k, harnessSubstitutions); if (ImageLayerBuildingSupport.buildingSharedLayer()) { ImageSingletonsSupportImpl.HostedManagement.persist(); - HostedImageLayerBuildingSupport.singleton().getWriter().dumpFile(); + HostedImageLayerBuildingSupport.singleton().archiveLayer(imageName); } } finally { reporter.ensureCreationStageEndCompleted(); @@ -796,15 +794,6 @@ protected void buildNativeImageHeap(NativeImageHeap heap, NativeImageCodeCache c heap.addTrailingObjects(); } - private static void setupImageLayerArtifact(String imageName) { - int imageNameStart = imageName.lastIndexOf('/') + 1; - String fileName = ImageLayerSnapshotUtil.FILE_NAME_PREFIX + imageName.substring(imageNameStart); - String filePath = imageName.substring(0, imageNameStart) + fileName; - Path layerSnapshotPath = generatedFiles(HostedOptionValues.singleton()).resolve(filePath + ImageLayerSnapshotUtil.FILE_EXTENSION); - HostedImageLayerBuildingSupport.singleton().getWriter().setFileInfo(layerSnapshotPath, fileName, ImageLayerSnapshotUtil.FILE_EXTENSION); - BuildArtifacts.singleton().add(ArtifactType.LAYER_SNAPSHOT, layerSnapshotPath); - } - protected void createAbstractImage(NativeImageKind k, List hostedEntryPoints, NativeImageHeap heap, HostedMetaAccess hMetaAccess, NativeImageCodeCache codeCache) { this.image = AbstractImage.create(k, hUniverse, hMetaAccess, nativeLibraries, heap, codeCache, hostedEntryPoints, loader.getClassLoader()); } @@ -1301,23 +1290,7 @@ protected NativeLibraries setupNativeLibraries(HostedProviders providers, CEnumC throw new InterruptImageBuilding("Exiting image generation because of " + SubstrateOptionsParser.commandArgument(CAnnotationProcessorCache.Options.ExitAfterCAPCache, "+")); } if (ImageLayerBuildingSupport.buildingExtensionLayer()) { - for (Path layerPath : HostedImageLayerBuildingSupport.singleton().getLoader().getLoadPaths()) { - Path snapshotFileName = layerPath.getFileName(); - if (snapshotFileName != null) { - String layerName = snapshotFileName.toString().split(ImageLayerSnapshotUtil.FILE_NAME_PREFIX)[1].split(ImageLayerSnapshotUtil.FILE_EXTENSION)[0].trim(); - /* - * This currently assumes lib{layer}.so is in the same dir as the layer - * snapshot. GR-53663 will create a proper bundle that contains both files. - */ - Path layerPathDir = layerPath.getParent(); - if (layerPathDir != null && layerName.startsWith("lib") && Files.exists(layerPathDir.resolve(layerName + ".so"))) { - nativeLibs.getLibraryPaths().add(layerPathDir.toString()); - nativeLibs.addDynamicNonJniLibrary(layerName.split("lib")[1]); - } else { - throw VMError.shouldNotReachHere("Missing " + layerName + ".so. It must be placed in the same dir as the layer snapshot."); - } - } - } + HostedImageLayerBuildingSupport.setupSharedLayerLibrary(nativeLibs); } return nativeLibs; } @@ -2004,6 +1977,10 @@ private static String slotsToString(short[] slots) { return result.toString(); } + public static Path getOutputDirectory() { + return NativeImageGenerator.generatedFiles(HostedOptionValues.singleton()); + } + public static Path generatedFiles(OptionValues optionValues) { String pathName = SubstrateOptions.Path.getValue(optionValues); Path path = FileSystems.getDefault().getPath(pathName); diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/NativeImageGeneratorRunner.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/NativeImageGeneratorRunner.java index be64d53db1e3..ea1b21fb7a0c 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/NativeImageGeneratorRunner.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/NativeImageGeneratorRunner.java @@ -397,13 +397,13 @@ private int buildImage(ImageClassLoader classLoader) { NativeImageKind imageKind = null; boolean isStaticExecutable = SubstrateOptions.StaticExecutable.getValue(parsedHostedOptions); boolean isSharedLibrary = SubstrateOptions.SharedLibrary.getValue(parsedHostedOptions); - boolean isImageLayer = SubstrateOptions.ImageLayer.getValue(parsedHostedOptions); + boolean isImageLayer = SubstrateOptions.LayerCreate.hasBeenSet(parsedHostedOptions); if (isStaticExecutable && isSharedLibrary) { reportConflictingOptions(SubstrateOptions.SharedLibrary, SubstrateOptions.StaticExecutable); } else if (isStaticExecutable && isImageLayer) { - reportConflictingOptions(SubstrateOptions.StaticExecutable, SubstrateOptions.ImageLayer); + reportConflictingOptions(SubstrateOptions.StaticExecutable, SubstrateOptions.LayerCreate); } else if (isSharedLibrary && isImageLayer) { - reportConflictingOptions(SubstrateOptions.SharedLibrary, SubstrateOptions.ImageLayer); + reportConflictingOptions(SubstrateOptions.SharedLibrary, SubstrateOptions.LayerCreate); } else if (isSharedLibrary) { imageKind = NativeImageKind.SHARED_LIBRARY; } else if (isImageLayer) { @@ -587,7 +587,7 @@ private int buildImage(ImageClassLoader classLoader) { return ExitStatus.OK.getValue(); } - private static void reportConflictingOptions(HostedOptionKey o1, HostedOptionKey o2) { + private static void reportConflictingOptions(HostedOptionKey o1, HostedOptionKey o2) { throw UserError.abort("Cannot pass both options: %s and %s", SubstrateOptionsParser.commandArgument(o1, "+"), SubstrateOptionsParser.commandArgument(o2, "+")); } diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/imagelayer/HostedImageLayerBuildingSupport.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/imagelayer/HostedImageLayerBuildingSupport.java index 1d5f1c2d7b70..633b8ef524a3 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/imagelayer/HostedImageLayerBuildingSupport.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/imagelayer/HostedImageLayerBuildingSupport.java @@ -24,22 +24,56 @@ */ package com.oracle.svm.hosted.imagelayer; -import com.oracle.svm.core.imagelayer.ImageLayerBuildingSupport; +import static com.oracle.svm.core.SubstrateOptions.LayerUse; +import static com.oracle.svm.core.SubstrateOptions.LayerCreate; +import static com.oracle.svm.core.SubstrateOptions.IncludeAllFromModule; +import static com.oracle.svm.core.SubstrateOptions.IncludeAllFromPath; +import static com.oracle.svm.core.SubstrateOptions.imageLayerEnabledHandler; +import static com.oracle.svm.hosted.imagelayer.LayerArchiveSupport.MODULE_OPTION; +import static com.oracle.svm.hosted.imagelayer.LayerArchiveSupport.PACKAGE_OPTION; + +import java.io.File; +import java.nio.file.Path; +import java.util.Arrays; +import java.util.List; + +import org.graalvm.collections.EconomicMap; import org.graalvm.nativeimage.ImageSingletons; +import com.oracle.graal.pointsto.heap.ImageLayerSnapshotUtil; +import com.oracle.svm.core.BuildArtifacts; import com.oracle.svm.core.SubstrateOptions; +import com.oracle.svm.core.imagelayer.ImageLayerBuildingSupport; +import com.oracle.svm.core.option.AccumulatingLocatableMultiOptionValue; +import com.oracle.svm.core.option.HostedOptionKey; import com.oracle.svm.core.option.HostedOptionValues; +import com.oracle.svm.core.option.SubstrateOptionsParser; +import com.oracle.svm.core.util.ArchiveSupport; +import com.oracle.svm.core.util.UserError; +import com.oracle.svm.core.util.VMError; +import com.oracle.svm.hosted.NativeImageGenerator; +import com.oracle.svm.hosted.c.NativeLibraries; import com.oracle.svm.hosted.heap.SVMImageLayerLoader; import com.oracle.svm.hosted.heap.SVMImageLayerWriter; +import com.oracle.svm.hosted.imagelayer.LayerOptionsSupport.LayerOption; + +import jdk.graal.compiler.core.common.SuppressFBWarnings; +import jdk.graal.compiler.options.OptionKey; +import jdk.graal.compiler.options.OptionValues; public final class HostedImageLayerBuildingSupport extends ImageLayerBuildingSupport { private final SVMImageLayerLoader loader; private final SVMImageLayerWriter writer; + private final WriteLayerArchiveSupport writeLayerArchiveSupport; + private final LoadLayerArchiveSupport loadLayerArchiveSupport; - private HostedImageLayerBuildingSupport(SVMImageLayerLoader loader, SVMImageLayerWriter writer, boolean buildingImageLayer, boolean buildingInitialLayer, boolean buildingApplicationLayer) { + private HostedImageLayerBuildingSupport(SVMImageLayerLoader loader, SVMImageLayerWriter writer, boolean buildingImageLayer, boolean buildingInitialLayer, boolean buildingApplicationLayer, + WriteLayerArchiveSupport writeLayerArchiveSupport, LoadLayerArchiveSupport loadLayerArchiveSupport) { super(buildingImageLayer, buildingInitialLayer, buildingApplicationLayer); this.loader = loader; this.writer = writer; + this.writeLayerArchiveSupport = writeLayerArchiveSupport; + this.loadLayerArchiveSupport = loadLayerArchiveSupport; } public static HostedImageLayerBuildingSupport singleton() { @@ -54,19 +88,130 @@ public SVMImageLayerWriter getWriter() { return writer; } + public LoadLayerArchiveSupport getLoadLayerArchiveSupport() { + return loadLayerArchiveSupport; + } + + public void archiveLayer(String imageName) { + writer.dumpFile(); + writeLayerArchiveSupport.write(imageName); + } + + /** + * Process layer-create and layer-use options. The semantics of these options allow a user to + * specify them any number of times, only the last instance wins. This processing cannot be done + * in {@code HostedOptionKey.onValueUpdate()} because processing this options affects other + * option's values, and any intermediate state may lead to a wrong configuration. + */ + public static void processLayerOptions(EconomicMap, Object> values) { + OptionValues hostedOptions = new OptionValues(values); + if (LayerCreate.hasBeenSet(hostedOptions)) { + /* The last value wins, GR-55565 will warn about the overwritten values. */ + String layerCreateValue = LayerCreate.getValue(hostedOptions).lastValue().orElseThrow(); + if (layerCreateValue.isEmpty()) { + /* Nothing to do, an empty --layer-create= disables the layer creation. */ + } else { + LayerOption layerOption = LayerOption.parse(layerCreateValue); + String buildLayer = SubstrateOptionsParser.commandArgument(LayerCreate, ""); + Arrays.stream(layerOption.extendedOptions()).forEach(option -> { + switch (option.key()) { + case MODULE_OPTION -> { + UserError.guarantee(option.value() != null, "Option %s of %s requires a module name argument, e.g., %s=module-name.", option.key(), buildLayer, option.key()); + IncludeAllFromModule.update(values, option.value()); + } + case PACKAGE_OPTION -> { + UserError.guarantee(option.value() != null, "Option %s of %s requires a package name argument, e.g., %s=package-name.", option.key(), buildLayer, option.key()); + IncludeAllFromPath.update(values, option.value()); + } + default -> + throw UserError.abort("Unknown option %s of %s. Use --help-extra for usage instructions.", option.key(), buildLayer); + } + }); + + SubstrateOptions.LayeredBaseImageAnalysis.update(values, true); + SubstrateOptions.ClosedTypeWorld.update(values, false); + SubstrateOptions.StripDebugInfo.update(values, false); + SubstrateOptions.AOTTrivialInline.update(values, false); + if (imageLayerEnabledHandler != null) { + imageLayerEnabledHandler.onOptionEnabled(values); + } + SubstrateOptions.UseContainerSupport.update(values, false); + } + } + if (LayerUse.hasBeenSet(hostedOptions)) { + /* The last value wins, GR-55565 will warn about the overwritten values. */ + String layerUseValue = LayerUse.getValue(hostedOptions).lastValue().orElseThrow(); + if (layerUseValue.isEmpty()) { + /* Nothing to do, an empty --layer-use= disables the layer application. */ + } else { + LayerOption layerOption = LayerOption.parse(LayerUse.getValue(hostedOptions).lastValue().orElseThrow()); + if (layerOption.fileName() == null) { + String optionName = SubstrateOptionsParser.commandArgument(LayerUse, ""); + throw UserError.abort("Option %s requires a layer file argument, e.g., %s=layer-file.nil.", optionName, optionName); + } + SubstrateOptions.ClosedTypeWorld.update(values, false); + /* Ignore any potential undefined references caused by inlining in base layer. */ + SubstrateOptions.IgnoreUndefinedReferences.update(values, true); + SubstrateOptions.AOTTrivialInline.update(values, false); + if (imageLayerEnabledHandler != null) { + imageLayerEnabledHandler.onOptionEnabled(values); + } + } + } + } + + private static boolean isEnabled(HostedOptionKey option, HostedOptionValues values) { + if (option.hasBeenSet(values)) { + String lastOptionValue = option.getValue(values).lastValue().orElseThrow(); + return !lastOptionValue.isEmpty(); + } + return false; + } + public static HostedImageLayerBuildingSupport initialize(HostedOptionValues values) { + WriteLayerArchiveSupport writeLayerArchiveSupport = null; SVMImageLayerWriter writer = null; - if (SubstrateOptions.ImageLayer.getValue(values)) { + ArchiveSupport archiveSupport = new ArchiveSupport(false); + if (isEnabled(LayerCreate, values)) { + LayerOption layerOption = LayerOption.parse(LayerCreate.getValue(values).lastValue().orElseThrow()); + writeLayerArchiveSupport = new WriteLayerArchiveSupport(archiveSupport, layerOption.fileName()); writer = new SVMImageLayerWriter(); } SVMImageLayerLoader loader = null; - if (SubstrateOptions.LoadImageLayer.hasBeenSet(values)) { - loader = new SVMImageLayerLoader(SubstrateOptions.LoadImageLayer.getValue(values).values()); + LoadLayerArchiveSupport loadLayerArchiveSupport = null; + if (isEnabled(LayerUse, values)) { + LayerOption layerOption = LayerOption.parse(LayerUse.getValue(values).lastValue().orElseThrow()); + loadLayerArchiveSupport = new LoadLayerArchiveSupport(layerOption.fileName(), archiveSupport); + loader = new SVMImageLayerLoader(List.of(loadLayerArchiveSupport.getSnapshotPath())); } boolean buildingImageLayer = loader != null || writer != null; boolean buildingInitialLayer = buildingImageLayer && loader == null; boolean buildingFinalLayer = buildingImageLayer && writer == null; - return new HostedImageLayerBuildingSupport(loader, writer, buildingImageLayer, buildingInitialLayer, buildingFinalLayer); + return new HostedImageLayerBuildingSupport(loader, writer, buildingImageLayer, buildingInitialLayer, buildingFinalLayer, writeLayerArchiveSupport, loadLayerArchiveSupport); + } + + @SuppressFBWarnings(value = "NP", justification = "FB reports null pointer dereferencing because it doesn't see through UserError.guarantee.") + public static void setupSharedLayerLibrary(NativeLibraries nativeLibs) { + Path sharedLibPath = HostedImageLayerBuildingSupport.singleton().getLoadLayerArchiveSupport().getSharedLibraryPath(); + Path parent = sharedLibPath.getParent(); + VMError.guarantee(parent != null, "Shared layer library path doesn't have a parent."); + nativeLibs.getLibraryPaths().add(parent.toString()); + Path fileName = sharedLibPath.getFileName(); + VMError.guarantee(fileName != null, "Cannot determine shared layer library file name."); + String libName = fileName.toString(); + VMError.guarantee(libName.startsWith("lib") && libName.endsWith(".so"), "Expecting that shared layer library file starts with lib and ends with .so. Found: %s", libName); + nativeLibs.addDynamicNonJniLibrary(libName.substring("lib".length(), libName.indexOf(".so"))); + } + + public static void setupImageLayerArtifact(String imageName) { + VMError.guarantee(!imageName.contains(File.separator), "Expected simple file name, found %s.", imageName); + Path snapshotFile = NativeImageGenerator.getOutputDirectory().resolve(ImageLayerSnapshotUtil.snapshotFileName(imageName)); + Path fileName = snapshotFile.getFileName(); + if (fileName == null) { + throw VMError.shouldNotReachHere("Layer snapshot file doesn't exist."); + } + HostedImageLayerBuildingSupport.singleton().getWriter().setFileInfo(snapshotFile, fileName.toString(), ImageLayerSnapshotUtil.FILE_EXTENSION); + BuildArtifacts.singleton().add(BuildArtifacts.ArtifactType.LAYER_SNAPSHOT, snapshotFile); } } diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/imagelayer/LayerArchiveSupport.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/imagelayer/LayerArchiveSupport.java new file mode 100644 index 000000000000..8d5406a148ce --- /dev/null +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/imagelayer/LayerArchiveSupport.java @@ -0,0 +1,168 @@ +/* + * Copyright (c) 2024, 2024, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package com.oracle.svm.hosted.imagelayer; + +import java.io.IOException; +import java.io.OutputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.HashMap; +import java.util.Locale; +import java.util.Map; +import java.util.Properties; + +import com.oracle.svm.core.OS; +import com.oracle.svm.core.SubstrateUtil; +import com.oracle.svm.core.util.ArchiveSupport; +import com.oracle.svm.core.util.UserError; +import com.oracle.svm.core.util.VMError; +import com.oracle.svm.hosted.NativeImageGenerator; +import com.oracle.svm.util.LogUtils; + +public class LayerArchiveSupport { + + protected static final String MODULE_OPTION = "module"; + protected static final String PACKAGE_OPTION = "package"; + + private static final int LAYER_FILE_FORMAT_VERSION_MAJOR = 0; + private static final int LAYER_FILE_FORMAT_VERSION_MINOR = 1; + + protected static final String LAYER_INFO_MESSAGE_PREFIX = "Native Image Layers"; + protected static final String LAYER_TEMP_DIR_PREFIX = "layerRoot-"; + + protected static final String LAYER_FILE_EXTENSION = ".nil"; + + protected final LayerProperties layerProperties; + protected final ArchiveSupport archiveSupport; + + public LayerArchiveSupport(ArchiveSupport archiveSupport) { + this.archiveSupport = archiveSupport; + this.layerProperties = new LayerArchiveSupport.LayerProperties(); + } + + protected static final Path layerPropertiesFileName = Path.of("META-INF/nilayer.properties"); + + public final class LayerProperties { + + private static final String PROPERTY_KEY_LAYER_FILE_VERSION_MAJOR = "LayerFileVersionMajor"; + private static final String PROPERTY_KEY_LAYER_FILE_VERSION_MINOR = "LayerFileVersionMinor"; + private static final String PROPERTY_KEY_LAYER_FILE_CREATION_TIMESTAMP = "LayerFileCreationTimestamp"; + private static final String PROPERTY_KEY_NATIVE_IMAGE_PLATFORM = "NativeImagePlatform"; + private static final String PROPERTY_KEY_NATIVE_IMAGE_VENDOR = "NativeImageVendor"; + private static final String PROPERTY_KEY_NATIVE_IMAGE_VERSION = "NativeImageVersion"; + private static final String PROPERTY_KEY_IMAGE_LAYER_NAME = "LayerName"; + + private final Map properties; + + LayerProperties() { + this.properties = new HashMap<>(); + } + + void loadAndVerify(Path inputLayerLocation, Path expandedInputLayerDir) { + Path layerFileName = inputLayerLocation.getFileName(); + Path layerPropertiesFile = expandedInputLayerDir.resolve(layerPropertiesFileName); + + if (!Files.isReadable(layerPropertiesFile)) { + throw UserError.abort("The given layer file " + layerFileName + " does not contain a layer properties file"); + } + + properties.putAll(ArchiveSupport.loadProperties(layerPropertiesFile)); + verifyVersion(layerFileName); + + String niVendor = properties.getOrDefault(PROPERTY_KEY_NATIVE_IMAGE_VENDOR, "unknown"); + String javaVmVendor = System.getProperty("java.vm.vendor"); + String currentVendor = niVendor.equals(javaVmVendor) ? "" : " != '" + javaVmVendor + "'"; + String niVersion = properties.getOrDefault(PROPERTY_KEY_NATIVE_IMAGE_VERSION, "unknown"); + String javaVmVersion = System.getProperty("java.vm.version"); + String currentVersion = niVersion.equals(javaVmVersion) ? "" : " != '" + javaVmVersion + "'"; + String niPlatform = properties.getOrDefault(PROPERTY_KEY_NATIVE_IMAGE_PLATFORM, "unknown"); + // GR-55581 will enforce platform compatibility + String currentPlatform = niPlatform.equals(platform()) ? "" : " != '" + platform() + "'"; + String layerCreationTimestamp = properties.getOrDefault(PROPERTY_KEY_LAYER_FILE_CREATION_TIMESTAMP, ""); + info("Loaded layer from %s", layerFileName); + info("Layer created at '%s'", ArchiveSupport.parseTimestamp(layerCreationTimestamp)); + info("Using version: '%s'%s (vendor '%s'%s) on platform: '%s'%s", niVersion, currentVersion, niVendor, currentVendor, niPlatform, currentPlatform); + } + + private void verifyVersion(Path layerFileName) { + String fileVersionKey = PROPERTY_KEY_LAYER_FILE_VERSION_MAJOR; + try { + int major = Integer.parseInt(properties.getOrDefault(fileVersionKey, "-1")); + fileVersionKey = PROPERTY_KEY_LAYER_FILE_VERSION_MINOR; + int minor = Integer.parseInt(properties.getOrDefault(fileVersionKey, "-1")); + String message = String.format("The given layer file %s was created with a newer layer-file-format version %d.%d" + + " (current %d.%d). Update to the latest version of native-image.", layerFileName, major, minor, LAYER_FILE_FORMAT_VERSION_MAJOR, LAYER_FILE_FORMAT_VERSION_MINOR); + if (major > LAYER_FILE_FORMAT_VERSION_MAJOR) { + throw UserError.abort(message); + } else if (major == LAYER_FILE_FORMAT_VERSION_MAJOR) { + if (minor > LAYER_FILE_FORMAT_VERSION_MINOR) { + LogUtils.warning(message); + } + } + } catch (NumberFormatException e) { + throw VMError.shouldNotReachHere(fileVersionKey + " in " + layerPropertiesFileName + " is missing or ill-defined", e); + } + } + + void write() { + properties.put(PROPERTY_KEY_LAYER_FILE_CREATION_TIMESTAMP, ArchiveSupport.currentTime()); + properties.put(PROPERTY_KEY_NATIVE_IMAGE_PLATFORM, platform()); + properties.put(PROPERTY_KEY_NATIVE_IMAGE_VENDOR, System.getProperty("java.vm.vendor")); + properties.put(PROPERTY_KEY_NATIVE_IMAGE_VERSION, System.getProperty("java.vm.version")); + Path layerPropertiesFile = NativeImageGenerator.getOutputDirectory().resolve(layerPropertiesFileName); + Path parent = layerPropertiesFile.getParent(); + if (parent == null) { + throw VMError.shouldNotReachHere("The layer properties file " + layerPropertiesFile + " doesn't have a parent directory."); + } + archiveSupport.ensureDirectoryExists(parent); + try (OutputStream outputStream = Files.newOutputStream(layerPropertiesFile)) { + Properties p = new Properties(); + p.putAll(properties); + p.store(outputStream, "Native Image Layer file properties"); + } catch (IOException e) { + throw VMError.shouldNotReachHere("Creating layer properties file " + layerPropertiesFile + " failed", e); + } + } + + public void writeLayerName(String layerName) { + properties.put(PROPERTY_KEY_IMAGE_LAYER_NAME, layerName); + } + + public String layerName() { + VMError.guarantee(!properties.isEmpty(), "Property file is no loaded."); + String name = properties.get(PROPERTY_KEY_IMAGE_LAYER_NAME); + VMError.guarantee(name != null, "Property " + PROPERTY_KEY_IMAGE_LAYER_NAME + " must be set."); + return name; + } + } + + private static String platform() { + return (OS.getCurrent().className + "-" + SubstrateUtil.getArchitectureName()).toLowerCase(Locale.ROOT); + } + + protected static void info(String format, Object... args) { + LogUtils.prefixInfo(LAYER_INFO_MESSAGE_PREFIX, format, args); + } +} diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/imagelayer/LayerOptionsSupport.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/imagelayer/LayerOptionsSupport.java new file mode 100644 index 000000000000..befcd98a9215 --- /dev/null +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/imagelayer/LayerOptionsSupport.java @@ -0,0 +1,65 @@ +/* + * Copyright (c) 2024, 2024, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package com.oracle.svm.hosted.imagelayer; + +import java.util.Arrays; + +import com.oracle.svm.core.SubstrateUtil; +import com.oracle.svm.core.util.VMError; + +public class LayerOptionsSupport { + + public record LayerOption(String fileName, ExtendedOption[] extendedOptions) { + /** Split a layer option into its components. */ + public static LayerOption parse(String layerOptionValue) { + VMError.guarantee(!layerOptionValue.isEmpty()); + // Given an argument of form layer-file.nil,module=m1,package=p1 + // First get the list: [layer-file.nil, module=m1, package=p1] + String[] options = SubstrateUtil.split(layerOptionValue, ","); + // Check for the optional file name + String fileName = null; + int skip = 0; + if (options[0].endsWith(LayerArchiveSupport.LAYER_FILE_EXTENSION)) { + fileName = options[0]; + skip = 1; + } + ExtendedOption[] extendedOptions = Arrays.stream(options).skip(skip).map(ExtendedOption::parse).toArray(ExtendedOption[]::new); + return new LayerOption(fileName, extendedOptions); + } + } + + public record ExtendedOption(String key, String value) { + + static ExtendedOption parse(String option) { + String[] optionParts = SubstrateUtil.split(option, "=", 2); + if (optionParts.length == 2) { + return new ExtendedOption(optionParts[0], optionParts[1]); + } else { + return new ExtendedOption(option, null); + } + } + } + +} diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/imagelayer/LoadLayerArchiveSupport.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/imagelayer/LoadLayerArchiveSupport.java new file mode 100644 index 000000000000..bf0deb394303 --- /dev/null +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/imagelayer/LoadLayerArchiveSupport.java @@ -0,0 +1,73 @@ +/* + * Copyright (c) 2024, 2024, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package com.oracle.svm.hosted.imagelayer; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.concurrent.atomic.AtomicBoolean; + +import com.oracle.graal.pointsto.heap.ImageLayerSnapshotUtil; +import com.oracle.svm.core.util.ArchiveSupport; +import com.oracle.svm.core.util.UserError; + +public class LoadLayerArchiveSupport extends LayerArchiveSupport { + + /** The temp directory where the input layer is expanded. */ + private final Path expandedInputLayerDir; + + private final AtomicBoolean deleteLayerRoot = new AtomicBoolean(); + + public LoadLayerArchiveSupport(String layerFile, ArchiveSupport archiveSupport) { + super(archiveSupport); + Path inputLayerLocation = validateLayerFile(layerFile); + expandedInputLayerDir = this.archiveSupport.createTempDir(LAYER_TEMP_DIR_PREFIX, deleteLayerRoot); + this.archiveSupport.expandJarToDir(inputLayerLocation, expandedInputLayerDir, deleteLayerRoot); + layerProperties.loadAndVerify(inputLayerLocation, expandedInputLayerDir); + } + + public Path getSharedLibraryPath() { + return expandedInputLayerDir.resolve(layerProperties.layerName() + ".so"); + } + + public Path getSnapshotPath() { + return expandedInputLayerDir.resolve(ImageLayerSnapshotUtil.snapshotFileName(layerProperties.layerName())); + } + + private static Path validateLayerFile(String layerFileArg) { + if (!layerFileArg.endsWith(LAYER_FILE_EXTENSION)) { + throw UserError.abort("The given layer file " + layerFileArg + " must end with '" + LAYER_FILE_EXTENSION + "'."); + } + Path layerFile = Path.of(layerFileArg); + Path layerFilePath = layerFile.toAbsolutePath(); + if (Files.isDirectory(layerFilePath)) { + throw UserError.abort("The given layer file " + layerFileArg + " is a directory and not a file."); + } + if (!Files.isReadable(layerFilePath)) { + throw UserError.abort("The given layer file " + layerFileArg + " cannot be read."); + } + return layerFile; + } + +} diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/imagelayer/WriteLayerArchiveSupport.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/imagelayer/WriteLayerArchiveSupport.java new file mode 100644 index 000000000000..6412aada30b3 --- /dev/null +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/imagelayer/WriteLayerArchiveSupport.java @@ -0,0 +1,93 @@ +/* + * Copyright (c) 2024, 2024, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package com.oracle.svm.hosted.imagelayer; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.jar.JarOutputStream; + +import com.oracle.svm.core.BuildArtifacts; +import com.oracle.svm.core.util.ArchiveSupport; +import com.oracle.svm.core.util.UserError; +import com.oracle.svm.hosted.NativeImageGenerator; + +/* Builds an image layer, either initial or intermediate. */ +public class WriteLayerArchiveSupport extends LayerArchiveSupport { + + /** The original location of the layer output file. */ + private final Path outputLayerLocation; + + public WriteLayerArchiveSupport(ArchiveSupport archiveSupport, String layerFile) { + super(archiveSupport); + this.outputLayerLocation = validateLayerFile(layerFile); + } + + private static Path validateLayerFile(String layerFileArg) { + if (!layerFileArg.endsWith(LAYER_FILE_EXTENSION)) { + throw UserError.abort("The given layer file " + layerFileArg + " must end with '" + LAYER_FILE_EXTENSION + "'."); + } + Path layerFile = Path.of(layerFileArg); + if (layerFile.getParent() != null) { + throw UserError.abort("The given layer file " + layerFileArg + " must be a simple file name, i.e., no path separators are allowed."); + } + Path layerFilePath = layerFile.toAbsolutePath(); + if (Files.isDirectory(layerFilePath)) { + throw UserError.abort("The given layer file " + layerFileArg + " is a directory and not a file."); + } + Path layerParentPath = layerFilePath.getParent(); + if (layerParentPath == null) { + throw UserError.abort("The given layer file " + layerFileArg + " doesn't have a parent directory."); + } + if (!Files.isWritable(layerParentPath)) { + throw UserError.abort("The layer file parent directory " + layerParentPath + " is not writeable."); + } + if (Files.exists(layerFilePath) && !Files.isWritable(layerFilePath)) { + throw UserError.abort("The given layer file " + layerFileArg + " is not writeable."); + } + return layerFile; + } + + public void write(String imageName) { + layerProperties.writeLayerName(String.valueOf(imageName)); + layerProperties.write(); + try (JarOutputStream jarOutStream = new JarOutputStream(Files.newOutputStream(outputLayerLocation), archiveSupport.createManifest())) { + Path imageBuilderOutputDir = NativeImageGenerator.getOutputDirectory(); + // copy the layer snapshot file to the jar + Path snapshotFile = BuildArtifacts.singleton().get(BuildArtifacts.ArtifactType.LAYER_SNAPSHOT).getFirst(); + archiveSupport.addFileToJar(imageBuilderOutputDir, snapshotFile, outputLayerLocation, jarOutStream); + // copy the shared object file to the jar + Path sharedLibFile = BuildArtifacts.singleton().get(BuildArtifacts.ArtifactType.IMAGE_LAYER).getFirst(); + archiveSupport.addFileToJar(imageBuilderOutputDir, sharedLibFile, outputLayerLocation, jarOutStream); + // copy the properties file to the jar + Path propertiesFile = imageBuilderOutputDir.resolve(layerPropertiesFileName); + archiveSupport.addFileToJar(imageBuilderOutputDir, propertiesFile, outputLayerLocation, jarOutStream); + } catch (IOException e) { + throw UserError.abort("Failed to create Native Image Layer file " + outputLayerLocation.getFileName(), e); + } + info("Layer written to %s", outputLayerLocation); + } + +} diff --git a/substratevm/src/com.oracle.svm.util/src/com/oracle/svm/util/LogUtils.java b/substratevm/src/com.oracle.svm.util/src/com/oracle/svm/util/LogUtils.java index a5d90f3d227b..2284cfe1277c 100644 --- a/substratevm/src/com.oracle.svm.util/src/com/oracle/svm/util/LogUtils.java +++ b/substratevm/src/com.oracle.svm.util/src/com/oracle/svm/util/LogUtils.java @@ -24,11 +24,11 @@ */ package com.oracle.svm.util; +import java.io.PrintStream; + import org.graalvm.nativeimage.Platform; import org.graalvm.nativeimage.Platforms; -import java.io.PrintStream; - // Checkstyle: Allow raw info or warning printing - begin public class LogUtils { /** @@ -58,6 +58,11 @@ public static void info(String format, Object... args) { info(format.formatted(args)); } + @Platforms(Platform.HOSTED_ONLY.class) + public static void prefixInfo(String prefix, String format, Object... args) { + info(prefix, format.formatted(args)); + } + /** * Print a warning. */