From 4cba141deeb5440c5bd16697f3ab5c5859ff1826 Mon Sep 17 00:00:00 2001 From: Robert Toyonaga Date: Wed, 25 Oct 2023 12:46:07 -0400 Subject: [PATCH 1/7] Use UnmanagedMemory instead of LibC directly. Add a method to release memory allocated outside java code. --- .../graalvm/nativeimage/UnmanagedMemory.java | 10 +++++ .../posix/PosixProcessPropertiesSupport.java | 6 +-- .../posix/UnmanagedMemorySupportImpl.java | 18 ++++++--- .../darwin/DarwinSystemPropertiesSupport.java | 6 +-- .../WindowsSystemPropertiesSupport.java | 37 ++++++++++--------- .../WindowsUnmanagedMemorySupportImpl.java | 18 ++++++--- .../src/com/oracle/svm/core/headers/LibC.java | 20 ---------- .../svm/core/jdk/TimeZoneSubstitutions.java | 2 +- 8 files changed, 62 insertions(+), 55 deletions(-) diff --git a/sdk/src/org.graalvm.nativeimage/src/org/graalvm/nativeimage/UnmanagedMemory.java b/sdk/src/org.graalvm.nativeimage/src/org/graalvm/nativeimage/UnmanagedMemory.java index a9d475d4ed0f..900580e9de51 100644 --- a/sdk/src/org.graalvm.nativeimage/src/org/graalvm/nativeimage/UnmanagedMemory.java +++ b/sdk/src/org.graalvm.nativeimage/src/org/graalvm/nativeimage/UnmanagedMemory.java @@ -146,4 +146,14 @@ public static T realloc(T ptr, UnsignedWord size) { public static void free(PointerBase ptr) { ImageSingletons.lookup(UnmanagedMemorySupport.class).free(ptr); } + + /** + * Temporarily the same as {@link UnmanagedMemory#free(PointerBase)}. Will later be different + * because it will not attempt to perform any NMT operations. This is crucial for releasing + * memory allocated by C libraries which will not have NMT "malloc headers". If + * {@link UnmanagedMemory#free(PointerBase)} is used instead, a segfault will occur. + */ + public static void untrackedFree(PointerBase ptr) { + ImageSingletons.lookup(UnmanagedMemorySupport.class).free(ptr); + } } diff --git a/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/PosixProcessPropertiesSupport.java b/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/PosixProcessPropertiesSupport.java index f3d2b586e41e..dfe66b33245d 100644 --- a/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/PosixProcessPropertiesSupport.java +++ b/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/PosixProcessPropertiesSupport.java @@ -36,12 +36,12 @@ import org.graalvm.nativeimage.c.type.CCharPointer; import org.graalvm.nativeimage.c.type.CTypeConversion; import org.graalvm.nativeimage.c.type.CTypeConversion.CCharPointerHolder; +import org.graalvm.nativeimage.UnmanagedMemory; import org.graalvm.word.PointerBase; import org.graalvm.word.WordFactory; import com.oracle.svm.core.BaseProcessPropertiesSupport; import com.oracle.svm.core.graal.stackvalue.UnsafeStackValue; -import com.oracle.svm.core.headers.LibC; import com.oracle.svm.core.posix.headers.Dlfcn; import com.oracle.svm.core.posix.headers.Signal; import com.oracle.svm.core.posix.headers.Stdlib; @@ -103,7 +103,7 @@ public String getObjectFile(PointerBase symbolAddress) { try { return CTypeConversion.toJavaString(realpath); } finally { - LibC.free(realpath); + UnmanagedMemory.untrackedFree(realpath); } } @@ -164,7 +164,7 @@ protected static String realpath(String path) { } else { /* Success */ final String result = CTypeConversion.toJavaString(realpathPointer); - LibC.free(realpathPointer); + UnmanagedMemory.untrackedFree(realpathPointer); return result; } } diff --git a/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/UnmanagedMemorySupportImpl.java b/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/UnmanagedMemorySupportImpl.java index 6c56453a7466..fb0d32d78bc5 100644 --- a/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/UnmanagedMemorySupportImpl.java +++ b/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/UnmanagedMemorySupportImpl.java @@ -24,6 +24,9 @@ */ package com.oracle.svm.core.posix; +import jdk.graal.compiler.api.replacements.Fold; + +import org.graalvm.nativeimage.ImageSingletons; import org.graalvm.nativeimage.impl.UnmanagedMemorySupport; import org.graalvm.word.PointerBase; import org.graalvm.word.UnsignedWord; @@ -31,31 +34,36 @@ import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.feature.AutomaticallyRegisteredImageSingleton; -import com.oracle.svm.core.headers.LibC; +import com.oracle.svm.core.headers.LibCSupport; @AutomaticallyRegisteredImageSingleton(UnmanagedMemorySupport.class) class UnmanagedMemorySupportImpl implements UnmanagedMemorySupport { @Override @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public T malloc(UnsignedWord size) { - return LibC.malloc(size); + return libc().malloc(size); } @Override @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public T calloc(UnsignedWord size) { - return LibC.calloc(WordFactory.unsigned(1), size); + return libc().calloc(WordFactory.unsigned(1), size); } @Override @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public T realloc(T ptr, UnsignedWord size) { - return LibC.realloc(ptr, size); + return libc().realloc(ptr, size); } @Override @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void free(PointerBase ptr) { - LibC.free(ptr); + libc().free(ptr); + } + + @Fold + static LibCSupport libc() { + return ImageSingletons.lookup(LibCSupport.class); } } diff --git a/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/darwin/DarwinSystemPropertiesSupport.java b/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/darwin/DarwinSystemPropertiesSupport.java index fe0e9ccbbc9a..88dc4551d9e3 100644 --- a/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/darwin/DarwinSystemPropertiesSupport.java +++ b/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/darwin/DarwinSystemPropertiesSupport.java @@ -31,13 +31,13 @@ import org.graalvm.nativeimage.c.type.CTypeConversion; import org.graalvm.nativeimage.c.type.CTypeConversion.CCharPointerHolder; import org.graalvm.nativeimage.impl.RuntimeSystemPropertiesSupport; +import org.graalvm.nativeimage.UnmanagedMemory; import org.graalvm.word.UnsignedWord; import org.graalvm.word.WordFactory; import com.oracle.svm.core.feature.AutomaticallyRegisteredFeature; import com.oracle.svm.core.feature.InternalFeature; import com.oracle.svm.core.graal.stackvalue.UnsafeStackValue; -import com.oracle.svm.core.headers.LibC; import com.oracle.svm.core.jdk.SystemPropertiesSupport; import com.oracle.svm.core.posix.PosixSystemPropertiesSupport; import com.oracle.svm.core.posix.headers.Limits; @@ -105,7 +105,7 @@ protected String osVersionValue() { CCharPointer osVersionStr = Foundation.systemVersionPlatform(); if (osVersionStr.isNonNull()) { osVersionValue = CTypeConversion.toJavaString(osVersionStr); - LibC.free(osVersionStr); + UnmanagedMemory.untrackedFree(osVersionStr); return osVersionValue; } } else { @@ -120,7 +120,7 @@ protected String osVersionValue() { CCharPointer osVersionStr = Foundation.systemVersionPlatformFallback(); if (osVersionStr.isNonNull()) { osVersionValue = CTypeConversion.toJavaString(osVersionStr); - LibC.free(osVersionStr); + UnmanagedMemory.untrackedFree(osVersionStr); return osVersionValue; } return osVersionValue = "Unknown"; diff --git a/substratevm/src/com.oracle.svm.core.windows/src/com/oracle/svm/core/windows/WindowsSystemPropertiesSupport.java b/substratevm/src/com.oracle.svm.core.windows/src/com/oracle/svm/core/windows/WindowsSystemPropertiesSupport.java index 353855263cba..2bff667d1bf9 100644 --- a/substratevm/src/com.oracle.svm.core.windows/src/com/oracle/svm/core/windows/WindowsSystemPropertiesSupport.java +++ b/substratevm/src/com.oracle.svm.core.windows/src/com/oracle/svm/core/windows/WindowsSystemPropertiesSupport.java @@ -38,6 +38,7 @@ import org.graalvm.nativeimage.c.type.VoidPointer; import org.graalvm.nativeimage.c.type.WordPointer; import org.graalvm.nativeimage.impl.RuntimeSystemPropertiesSupport; +import org.graalvm.nativeimage.impl.UnmanagedMemorySupport; import org.graalvm.word.UnsignedWord; import org.graalvm.word.WordFactory; @@ -45,7 +46,6 @@ import com.oracle.svm.core.feature.AutomaticallyRegisteredFeature; import com.oracle.svm.core.feature.InternalFeature; import com.oracle.svm.core.graal.stackvalue.UnsafeStackValue; -import com.oracle.svm.core.headers.LibC; import com.oracle.svm.core.jdk.SystemPropertiesSupport; import com.oracle.svm.core.util.VMError; import com.oracle.svm.core.windows.headers.FileAPI; @@ -245,29 +245,30 @@ public Pair getOsNameAndVersion() { break; } - VoidPointer versionInfo = LibC.malloc(WordFactory.unsigned(versionSize)); + VoidPointer versionInfo = ImageSingletons.lookup(UnmanagedMemorySupport.class).malloc(WordFactory.unsigned(versionSize)); if (versionInfo.isNull()) { break; } + try { - if (WinVer.GetFileVersionInfoW(kernel32Path, 0, versionSize, versionInfo) == 0) { - LibC.free(versionInfo); - break; - } + if (WinVer.GetFileVersionInfoW(kernel32Path, 0, versionSize, versionInfo) == 0) { + break; + } - WindowsLibC.WCharPointer rootPath = NonmovableArrays.addressOf(NonmovableArrays.fromImageHeap(ROOT_PATH), 0); - WordPointer fileInfoPointer = UnsafeStackValue.get(WordPointer.class); - CIntPointer lengthPointer = UnsafeStackValue.get(CIntPointer.class); - if (WinVer.VerQueryValueW(versionInfo, rootPath, fileInfoPointer, lengthPointer) == 0) { - LibC.free(versionInfo); - break; - } + WindowsLibC.WCharPointer rootPath = NonmovableArrays.addressOf(NonmovableArrays.fromImageHeap(ROOT_PATH), 0); + WordPointer fileInfoPointer = UnsafeStackValue.get(WordPointer.class); + CIntPointer lengthPointer = UnsafeStackValue.get(CIntPointer.class); + if (WinVer.VerQueryValueW(versionInfo, rootPath, fileInfoPointer, lengthPointer) == 0) { + break; + } - VerRsrc.VS_FIXEDFILEINFO fileInfo = fileInfoPointer.read(); - majorVersion = (short) (fileInfo.dwProductVersionMS() >> 16); // HIWORD - minorVersion = (short) fileInfo.dwProductVersionMS(); // LOWORD - buildNumber = (short) (fileInfo.dwProductVersionLS() >> 16); // HIWORD - LibC.free(versionInfo); + VerRsrc.VS_FIXEDFILEINFO fileInfo = fileInfoPointer.read(); + majorVersion = (short) (fileInfo.dwProductVersionMS() >> 16); // HIWORD + minorVersion = (short) fileInfo.dwProductVersionMS(); // LOWORD + buildNumber = (short) (fileInfo.dwProductVersionLS() >> 16); // HIWORD + } finally { + ImageSingletons.lookup(UnmanagedMemorySupport.class).free(versionInfo); + } } while (false); String osVersion = majorVersion + "." + minorVersion; diff --git a/substratevm/src/com.oracle.svm.core.windows/src/com/oracle/svm/core/windows/WindowsUnmanagedMemorySupportImpl.java b/substratevm/src/com.oracle.svm.core.windows/src/com/oracle/svm/core/windows/WindowsUnmanagedMemorySupportImpl.java index 7ee93b64d4e7..8c3d9933fcc3 100644 --- a/substratevm/src/com.oracle.svm.core.windows/src/com/oracle/svm/core/windows/WindowsUnmanagedMemorySupportImpl.java +++ b/substratevm/src/com.oracle.svm.core.windows/src/com/oracle/svm/core/windows/WindowsUnmanagedMemorySupportImpl.java @@ -24,6 +24,9 @@ */ package com.oracle.svm.core.windows; +import jdk.graal.compiler.api.replacements.Fold; + +import org.graalvm.nativeimage.ImageSingletons; import org.graalvm.nativeimage.impl.UnmanagedMemorySupport; import org.graalvm.word.PointerBase; import org.graalvm.word.UnsignedWord; @@ -31,31 +34,36 @@ import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.feature.AutomaticallyRegisteredImageSingleton; -import com.oracle.svm.core.headers.LibC; +import com.oracle.svm.core.headers.LibCSupport; @AutomaticallyRegisteredImageSingleton(UnmanagedMemorySupport.class) class WindowsUnmanagedMemorySupportImpl implements UnmanagedMemorySupport { @Override @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public T malloc(UnsignedWord size) { - return LibC.malloc(size); + return libc().malloc(size); } @Override @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public T calloc(UnsignedWord size) { - return LibC.calloc(WordFactory.unsigned(1), size); + return libc().calloc(WordFactory.unsigned(1), size); } @Override @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public T realloc(T ptr, UnsignedWord size) { - return LibC.realloc(ptr, size); + return libc().realloc(ptr, size); } @Override @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void free(PointerBase ptr) { - LibC.free(ptr); + libc().free(ptr); + } + + @Fold + static LibCSupport libc() { + return ImageSingletons.lookup(LibCSupport.class); } } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/headers/LibC.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/headers/LibC.java index b72147d4a989..f23f371b99e1 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/headers/LibC.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/headers/LibC.java @@ -67,26 +67,6 @@ public static T memset(T s, SignedWord c, UnsignedWord n return libc().memset(s, c, n); } - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - public static T malloc(UnsignedWord size) { - return libc().malloc(size); - } - - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - public static T calloc(UnsignedWord nmemb, UnsignedWord size) { - return libc().calloc(nmemb, size); - } - - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - public static T realloc(PointerBase ptr, UnsignedWord size) { - return libc().realloc(ptr, size); - } - - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - public static void free(PointerBase ptr) { - libc().free(ptr); - } - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static void exit(int status) { libc().exit(status); diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/TimeZoneSubstitutions.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/TimeZoneSubstitutions.java index 6875c685d0c2..6f3fb6cf8741 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/TimeZoneSubstitutions.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/TimeZoneSubstitutions.java @@ -100,7 +100,7 @@ private static String getSystemTimeZoneID(String javaHome) { CCharPointer tzId = LibCHelper.SVM_FindJavaTZmd(tzMappingsPtr, contentLen); String result = CTypeConversion.toJavaString(tzId); // SVM_FindJavaTZmd returns a newly allocated string - UnmanagedMemory.free(tzId); + UnmanagedMemory.untrackedFree(tzId); return result; } finally { if (refContent != null) { From a93c51ddc8d682afc01189eeb7a6226f9af75f79 Mon Sep 17 00:00:00 2001 From: Robert Toyonaga Date: Tue, 31 Oct 2023 16:57:05 -0400 Subject: [PATCH 2/7] malloc and virtual memory aggregated. Nmt feature. use stack for heap set up. add missing files --- .../graalvm/nativeimage/UnmanagedMemory.java | 2 +- .../impl/UnmanagedMemorySupport.java | 4 + .../posix/PosixVirtualMemoryProvider.java | 61 +++++++- .../posix/UnmanagedMemorySupportImpl.java | 40 ++++- .../posix/linux/LinuxImageHeapProvider.java | 19 ++- .../windows/WindowsImageHeapProvider.java | 9 +- .../WindowsUnmanagedMemorySupportImpl.java | 33 +++- .../windows/WindowsVirtualMemoryProvider.java | 36 ++++- .../src/com/oracle/svm/core/Isolates.java | 5 +- .../oracle/svm/core/VMInspectionOptions.java | 14 +- .../graal/snippets/CEntryPointSnippets.java | 17 +- .../oracle/svm/core/jfr/JfrChunkWriter.java | 1 + .../oracle/svm/core/nmt/HasNmtSupport.java | 43 ++++++ .../com/oracle/svm/core/nmt/MallocHeader.java | 46 ++++++ .../oracle/svm/core/nmt/MallocMemoryInfo.java | 65 ++++++++ .../svm/core/nmt/MallocMemorySnapshot.java | 56 +++++++ .../svm/core/nmt/NativeMemoryTracking.java | 145 ++++++++++++++++++ .../com/oracle/svm/core/nmt/NmtFeature.java | 46 ++++++ .../src/com/oracle/svm/core/nmt/NmtFlag.java | 45 ++++++ .../svm/core/nmt/NmtVirtualMemoryData.java | 47 ++++++ .../svm/core/nmt/VirtualMemoryInfo.java | 74 +++++++++ .../svm/core/nmt/VirtualMemorySnapshot.java | 58 +++++++ .../os/AbstractCopyingImageHeapProvider.java | 11 +- .../svm/core/os/CommittedMemoryProvider.java | 3 +- .../oracle/svm/core/os/ImageHeapProvider.java | 3 +- .../core/os/OSCommittedMemoryProvider.java | 5 +- .../svm/core/os/VirtualMemoryProvider.java | 6 + .../native-image.properties | 2 +- 28 files changed, 858 insertions(+), 38 deletions(-) create mode 100644 substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/HasNmtSupport.java create mode 100644 substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/MallocHeader.java create mode 100644 substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/MallocMemoryInfo.java create mode 100644 substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/MallocMemorySnapshot.java create mode 100644 substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/NativeMemoryTracking.java create mode 100644 substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/NmtFeature.java create mode 100644 substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/NmtFlag.java create mode 100644 substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/NmtVirtualMemoryData.java create mode 100644 substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/VirtualMemoryInfo.java create mode 100644 substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/VirtualMemorySnapshot.java diff --git a/sdk/src/org.graalvm.nativeimage/src/org/graalvm/nativeimage/UnmanagedMemory.java b/sdk/src/org.graalvm.nativeimage/src/org/graalvm/nativeimage/UnmanagedMemory.java index 900580e9de51..bcc7f962e262 100644 --- a/sdk/src/org.graalvm.nativeimage/src/org/graalvm/nativeimage/UnmanagedMemory.java +++ b/sdk/src/org.graalvm.nativeimage/src/org/graalvm/nativeimage/UnmanagedMemory.java @@ -154,6 +154,6 @@ public static void free(PointerBase ptr) { * {@link UnmanagedMemory#free(PointerBase)} is used instead, a segfault will occur. */ public static void untrackedFree(PointerBase ptr) { - ImageSingletons.lookup(UnmanagedMemorySupport.class).free(ptr); + ImageSingletons.lookup(UnmanagedMemorySupport.class).untrackedFree(ptr); } } diff --git a/sdk/src/org.graalvm.nativeimage/src/org/graalvm/nativeimage/impl/UnmanagedMemorySupport.java b/sdk/src/org.graalvm.nativeimage/src/org/graalvm/nativeimage/impl/UnmanagedMemorySupport.java index c18ab87631e8..964ac7b912ac 100644 --- a/sdk/src/org.graalvm.nativeimage/src/org/graalvm/nativeimage/impl/UnmanagedMemorySupport.java +++ b/sdk/src/org.graalvm.nativeimage/src/org/graalvm/nativeimage/impl/UnmanagedMemorySupport.java @@ -46,10 +46,14 @@ /** Implemented by operating-system specific code. */ public interface UnmanagedMemorySupport { T malloc(UnsignedWord size); + T malloc(UnsignedWord size, int flag); T calloc(UnsignedWord size); + T calloc(UnsignedWord size, int flag); T realloc(T ptr, UnsignedWord size); + T realloc(T ptr, UnsignedWord size, int flag); void free(PointerBase ptr); + void untrackedFree(PointerBase ptr); } diff --git a/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/PosixVirtualMemoryProvider.java b/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/PosixVirtualMemoryProvider.java index cffcbbb4550d..1b5868a0244a 100644 --- a/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/PosixVirtualMemoryProvider.java +++ b/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/PosixVirtualMemoryProvider.java @@ -54,6 +54,9 @@ import com.oracle.svm.core.c.CGlobalDataFactory; import com.oracle.svm.core.feature.AutomaticallyRegisteredFeature; import com.oracle.svm.core.feature.InternalFeature; +import com.oracle.svm.core.nmt.NativeMemoryTracking; +import com.oracle.svm.core.nmt.NmtFlag; +import com.oracle.svm.core.nmt.NmtVirtualMemoryData; import com.oracle.svm.core.os.VirtualMemoryProvider; import com.oracle.svm.core.posix.headers.Unistd; import com.oracle.svm.core.util.PointerUtils; @@ -110,6 +113,11 @@ public UnsignedWord getGranularity() { @Override @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public Pointer reserve(UnsignedWord nbytes, UnsignedWord alignment, boolean executable) { + return reserve(nbytes, alignment, executable, WordFactory.nullPointer()); + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public Pointer reserve(UnsignedWord nbytes, UnsignedWord alignment, boolean executable, NmtVirtualMemoryData nmtData) { if (nbytes.equal(0)) { return WordFactory.nullPointer(); } @@ -128,18 +136,32 @@ public Pointer reserve(UnsignedWord nbytes, UnsignedWord alignment, boolean exec return nullPointer(); } if (!customAlignment) { + if (nmtData.isNull()) { + NativeMemoryTracking.recordReserve(mappingSize, NmtFlag.Default.ordinal()); + } else { + nmtData.setReserved(nmtData.getReserved().add(mappingSize)); + } return mappingBegin; } + UnsignedWord unmappedSize = WordFactory.zero(); Pointer begin = PointerUtils.roundUp(mappingBegin, alignment); UnsignedWord clippedBegin = begin.subtract(mappingBegin); if (clippedBegin.aboveOrEqual(granularity)) { - munmap(mappingBegin, UnsignedUtils.roundDown(clippedBegin, granularity)); + UnsignedWord unmapSize = UnsignedUtils.roundDown(clippedBegin, granularity); + munmap(mappingBegin, unmapSize); + unmappedSize.add(unmapSize); } Pointer mappingEnd = mappingBegin.add(mappingSize); UnsignedWord clippedEnd = mappingEnd.subtract(begin.add(nbytes)); if (clippedEnd.aboveOrEqual(granularity)) { UnsignedWord rounded = UnsignedUtils.roundDown(clippedEnd, granularity); munmap(mappingEnd.subtract(rounded), rounded); + unmappedSize.add(rounded); + } + if (nmtData.isNull()) { + NativeMemoryTracking.recordReserve(mappingSize.subtract(unmappedSize), NmtFlag.Default.ordinal()); + } else { + nmtData.setReserved(nmtData.getReserved().add(mappingSize.subtract(unmappedSize))); } return begin; } @@ -147,6 +169,12 @@ public Pointer reserve(UnsignedWord nbytes, UnsignedWord alignment, boolean exec @Override @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public Pointer mapFile(PointerBase start, UnsignedWord nbytes, WordBase fileHandle, UnsignedWord offset, int access) { + return mapFile(start, nbytes, fileHandle, offset, access, WordFactory.nullPointer()); + } + + @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public Pointer mapFile(PointerBase start, UnsignedWord nbytes, WordBase fileHandle, UnsignedWord offset, int access, NmtVirtualMemoryData nmtData) { if ((start.isNonNull() && !isAligned(start)) || nbytes.equal(0)) { return WordFactory.nullPointer(); } @@ -157,12 +185,28 @@ public Pointer mapFile(PointerBase start, UnsignedWord nbytes, WordBase fileHand } int fd = (int) fileHandle.rawValue(); Pointer result = mmap(start, nbytes, accessAsProt(access), flags, fd, offset.rawValue()); - return result.notEqual(MAP_FAILED()) ? result : WordFactory.nullPointer(); + if (result.notEqual(MAP_FAILED())) { + if (nmtData.isNull()) { + // TODO is this also a reserve? [No, the mem has already been reserved in a previous + // call.] + NativeMemoryTracking.recordCommit(nbytes, NmtFlag.Default.ordinal()); + } else { + nmtData.setCommitted(nmtData.getCommitted().add(nbytes)); + } + return result; + } + return WordFactory.nullPointer(); } @Override @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public Pointer commit(PointerBase start, UnsignedWord nbytes, int access) { + return commit(start, nbytes, access, WordFactory.nullPointer()); + } + + @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public Pointer commit(PointerBase start, UnsignedWord nbytes, int access, NmtVirtualMemoryData nmtData) { if ((start.isNonNull() && !isAligned(start)) || nbytes.equal(0)) { return WordFactory.nullPointer(); } @@ -177,7 +221,15 @@ public Pointer commit(PointerBase start, UnsignedWord nbytes, int access) { } /* The memory returned by mmap is guaranteed to be zeroed. */ final Pointer result = mmap(start, nbytes, accessAsProt(access), flags, NO_FD, NO_FD_OFFSET); - return result.notEqual(MAP_FAILED()) ? result : nullPointer(); + if (result.notEqual(MAP_FAILED())) { + if (nmtData.isNull()) { + NativeMemoryTracking.recordCommit(nbytes, NmtFlag.Default.ordinal()); + } else { + nmtData.setCommitted(nmtData.getCommitted().add(nbytes)); + } + return result; + } + return nullPointer(); } @Override @@ -196,7 +248,7 @@ public int uncommit(PointerBase start, UnsignedWord nbytes) { if (start.isNull() || !isAligned(start) || nbytes.equal(0)) { return -1; } - + NativeMemoryTracking.recordUncommit(nbytes, NmtFlag.Default.ordinal()); final Pointer result = mmap(start, nbytes, PROT_NONE(), MAP_FIXED() | MAP_ANON() | MAP_PRIVATE() | MAP_NORESERVE(), NO_FD, NO_FD_OFFSET); return result.notEqual(MAP_FAILED()) ? 0 : -1; } @@ -211,6 +263,7 @@ public int free(PointerBase start, UnsignedWord nbytes) { UnsignedWord granularity = getGranularity(); Pointer mappingBegin = PointerUtils.roundDown(start, granularity); UnsignedWord mappingSize = UnsignedUtils.roundUp(nbytes, granularity); + NativeMemoryTracking.recordFree(nbytes, NmtFlag.Default.ordinal()); return munmap(mappingBegin, mappingSize); } diff --git a/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/UnmanagedMemorySupportImpl.java b/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/UnmanagedMemorySupportImpl.java index fb0d32d78bc5..b5cc431af6a8 100644 --- a/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/UnmanagedMemorySupportImpl.java +++ b/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/UnmanagedMemorySupportImpl.java @@ -25,40 +25,72 @@ package com.oracle.svm.core.posix; import jdk.graal.compiler.api.replacements.Fold; - import org.graalvm.nativeimage.ImageSingletons; import org.graalvm.nativeimage.impl.UnmanagedMemorySupport; + import org.graalvm.word.PointerBase; +import org.graalvm.word.Pointer; import org.graalvm.word.UnsignedWord; import org.graalvm.word.WordFactory; import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.feature.AutomaticallyRegisteredImageSingleton; import com.oracle.svm.core.headers.LibCSupport; +import com.oracle.svm.core.nmt.NmtFlag; +import com.oracle.svm.core.nmt.NativeMemoryTracking; @AutomaticallyRegisteredImageSingleton(UnmanagedMemorySupport.class) class UnmanagedMemorySupportImpl implements UnmanagedMemorySupport { @Override @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public T malloc(UnsignedWord size) { - return libc().malloc(size); + return malloc(size, NmtFlag.Default.ordinal()); + } + + @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public T malloc(UnsignedWord size, int flag) { + Pointer outerPointer = libc().malloc(size.add(NativeMemoryTracking.getHeaderSize())); + return (T) NativeMemoryTracking.recordMalloc(outerPointer, size, flag); } @Override @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public T calloc(UnsignedWord size) { - return libc().calloc(WordFactory.unsigned(1), size); + return calloc(size, NmtFlag.Default.ordinal()); + } + + @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public T calloc(UnsignedWord size, int flag) { + Pointer outerPointer = libc().calloc(WordFactory.unsigned(1), size.add(NativeMemoryTracking.getHeaderSize())); + return (T) NativeMemoryTracking.recordMalloc(outerPointer, size, flag); } @Override @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public T realloc(T ptr, UnsignedWord size) { - return libc().realloc(ptr, size); + return realloc(ptr, size, NmtFlag.Default.ordinal()); + } + + @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public T realloc(T ptr, UnsignedWord size, int flag) { + Pointer outerPointer = libc().realloc(ptr, size.add(NativeMemoryTracking.getHeaderSize())); + return (T) NativeMemoryTracking.recordMalloc(outerPointer, size, flag); } @Override @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void free(PointerBase ptr) { + NativeMemoryTracking.deaccountMalloc(ptr); + // *** tried confirming by changing this. It fails, as expected + libc().free(((Pointer) ptr).subtract(NativeMemoryTracking.getHeaderSize())); + } + + @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public void untrackedFree(PointerBase ptr) { libc().free(ptr); } diff --git a/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/linux/LinuxImageHeapProvider.java b/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/linux/LinuxImageHeapProvider.java index 555a381ff406..ce1f84f817cf 100644 --- a/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/linux/LinuxImageHeapProvider.java +++ b/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/linux/LinuxImageHeapProvider.java @@ -58,6 +58,7 @@ import com.oracle.svm.core.config.ConfigurationValues; import com.oracle.svm.core.headers.LibC; import com.oracle.svm.core.heap.Heap; +import com.oracle.svm.core.nmt.NmtVirtualMemoryData; import com.oracle.svm.core.os.AbstractImageHeapProvider; import com.oracle.svm.core.os.CopyingImageHeapProvider; import com.oracle.svm.core.os.VirtualMemoryProvider; @@ -104,7 +105,7 @@ public boolean guaranteesHeapPreferredAddressSpaceAlignment() { @Override @Uninterruptible(reason = "Called during isolate initialization.") - public int initialize(Pointer reservedAddressSpace, UnsignedWord reservedSize, WordPointer basePointer, WordPointer endPointer) { + public int initialize(Pointer reservedAddressSpace, UnsignedWord reservedSize, WordPointer basePointer, WordPointer endPointer, NmtVirtualMemoryData nmtData) { // If we are the first isolate, we might be able to use the existing image heap (see below) SignedWord fd = CACHED_IMAGE_FD.get().read(); boolean firstIsolate = false; @@ -140,7 +141,7 @@ public int initialize(Pointer reservedAddressSpace, UnsignedWord reservedSize, W // If we cannot find or open the image file, fall back to copy it from memory. if (fd.equal(CANNOT_OPEN_FD)) { - return fallbackCopyingProvider.initialize(reservedAddressSpace, reservedSize, basePointer, endPointer); + return fallbackCopyingProvider.initialize(reservedAddressSpace, reservedSize, basePointer, endPointer, nmtData); } boolean haveDynamicMethodResolution = DynamicMethodAddressResolutionHeapSupport.isEnabled(); @@ -198,7 +199,7 @@ public int initialize(Pointer reservedAddressSpace, UnsignedWord reservedSize, W Pointer heapBase; Pointer allocatedMemory = WordFactory.nullPointer(); if (reservedAddressSpace.isNull()) { - heapBase = allocatedMemory = VirtualMemoryProvider.get().reserve(totalAddressSpaceSize, alignment, false); + heapBase = allocatedMemory = VirtualMemoryProvider.get().reserve(totalAddressSpaceSize, alignment, false, nmtData); if (allocatedMemory.isNull()) { return CEntryPointErrors.RESERVE_ADDRESS_SPACE_FAILED; } @@ -226,7 +227,7 @@ public int initialize(Pointer reservedAddressSpace, UnsignedWord reservedSize, W // Create memory mappings from the image file. UnsignedWord fileOffset = CACHED_IMAGE_HEAP_OFFSET.get().read(); Pointer imageHeap = heapBase.add(imageHeapOffsetInAddressSpace); - imageHeap = VirtualMemoryProvider.get().mapFile(imageHeap, imageHeapSizeInFile, fd, fileOffset, Access.READ); + imageHeap = VirtualMemoryProvider.get().mapFile(imageHeap, imageHeapSizeInFile, fd, fileOffset, Access.READ, nmtData); if (imageHeap.isNull()) { freeImageHeap(allocatedMemory); return CEntryPointErrors.MAP_HEAP_FAILED; @@ -234,8 +235,16 @@ public int initialize(Pointer reservedAddressSpace, UnsignedWord reservedSize, W Pointer relocPointer = IMAGE_HEAP_A_RELOCATABLE_POINTER.get(); ComparableWord relocatedValue = relocPointer.readWord(0); - ComparableWord mappedValue = imageHeap.readWord(relocPointer.subtract(imageHeapBegin)); + ComparableWord mappedValue = imageHeap.readWord(relocPointer.subtract(imageHeapBegin)); // *** + // What + // is + // this + // mappedValue? if (relocatedValue.notEqual(mappedValue)) { + // TODO what is actually happening in this block? + // *** this is checking to see if they've been mapped correctly? [GDB: this doesn't + // actually get entered it seems] + /* * Addresses were relocated by dynamic linker, so copy them, but first remap the pages * to avoid swapping them in from disk. diff --git a/substratevm/src/com.oracle.svm.core.windows/src/com/oracle/svm/core/windows/WindowsImageHeapProvider.java b/substratevm/src/com.oracle.svm.core.windows/src/com/oracle/svm/core/windows/WindowsImageHeapProvider.java index 5b06c4420457..a7195bbe5672 100644 --- a/substratevm/src/com.oracle.svm.core.windows/src/com/oracle/svm/core/windows/WindowsImageHeapProvider.java +++ b/substratevm/src/com.oracle.svm.core.windows/src/com/oracle/svm/core/windows/WindowsImageHeapProvider.java @@ -47,6 +47,7 @@ import com.oracle.svm.core.c.function.CEntryPointErrors; import com.oracle.svm.core.headers.LibC; import com.oracle.svm.core.os.AbstractCopyingImageHeapProvider; +import com.oracle.svm.core.nmt.NmtVirtualMemoryData; import com.oracle.svm.core.os.VirtualMemoryProvider; import com.oracle.svm.core.os.VirtualMemoryProvider.Access; import com.oracle.svm.core.windows.headers.FileAPI; @@ -64,18 +65,18 @@ public class WindowsImageHeapProvider extends AbstractCopyingImageHeapProvider { @Override @Uninterruptible(reason = "Called during isolate initialization.") - protected int commitAndCopyMemory(Pointer loadedImageHeap, UnsignedWord imageHeapSize, Pointer newImageHeap) { + protected int commitAndCopyMemory(Pointer loadedImageHeap, UnsignedWord imageHeapSize, Pointer newImageHeap, NmtVirtualMemoryData nmtData) { HANDLE imageHeapFileMapping = getImageHeapFileMapping(); if (imageHeapFileMapping.isNull()) { /* Fall back to copying from memory. */ - return super.commitAndCopyMemory(loadedImageHeap, imageHeapSize, newImageHeap); + return super.commitAndCopyMemory(loadedImageHeap, imageHeapSize, newImageHeap, nmtData); } /* Map a copy-on-write view of the image heap. */ if (VirtualMemoryProvider.get().mapFile(newImageHeap, imageHeapSize, imageHeapFileMapping, getImageHeapFileOffset(), - Access.READ | Access.WRITE).isNull()) { + Access.READ | Access.WRITE, nmtData).isNull()) { /* Fall back to copying from memory. */ - return super.commitAndCopyMemory(loadedImageHeap, imageHeapSize, newImageHeap); + return super.commitAndCopyMemory(loadedImageHeap, imageHeapSize, newImageHeap, nmtData); } /* Copy relocatable pages. */ diff --git a/substratevm/src/com.oracle.svm.core.windows/src/com/oracle/svm/core/windows/WindowsUnmanagedMemorySupportImpl.java b/substratevm/src/com.oracle.svm.core.windows/src/com/oracle/svm/core/windows/WindowsUnmanagedMemorySupportImpl.java index 8c3d9933fcc3..4e296e5f446b 100644 --- a/substratevm/src/com.oracle.svm.core.windows/src/com/oracle/svm/core/windows/WindowsUnmanagedMemorySupportImpl.java +++ b/substratevm/src/com.oracle.svm.core.windows/src/com/oracle/svm/core/windows/WindowsUnmanagedMemorySupportImpl.java @@ -29,36 +29,63 @@ import org.graalvm.nativeimage.ImageSingletons; import org.graalvm.nativeimage.impl.UnmanagedMemorySupport; import org.graalvm.word.PointerBase; +import org.graalvm.word.Pointer; import org.graalvm.word.UnsignedWord; import org.graalvm.word.WordFactory; import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.feature.AutomaticallyRegisteredImageSingleton; import com.oracle.svm.core.headers.LibCSupport; +import com.oracle.svm.core.nmt.NmtFlag; +import com.oracle.svm.core.nmt.NativeMemoryTracking; @AutomaticallyRegisteredImageSingleton(UnmanagedMemorySupport.class) class WindowsUnmanagedMemorySupportImpl implements UnmanagedMemorySupport { @Override @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public T malloc(UnsignedWord size) { - return libc().malloc(size); + return malloc(size, NmtFlag.Default.ordinal()); + } + + @Override + public T malloc(UnsignedWord size, int flag) { + Pointer outerPointer = libc().malloc(size.add(NativeMemoryTracking.getHeaderSize())); + return (T) NativeMemoryTracking.recordMalloc(outerPointer, size, flag); } @Override @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public T calloc(UnsignedWord size) { - return libc().calloc(WordFactory.unsigned(1), size); + return calloc(size, NmtFlag.Default.ordinal()); + } + + @Override + public T calloc(UnsignedWord size, int flag) { + Pointer outerPointer = libc().calloc(WordFactory.unsigned(1), size.add(NativeMemoryTracking.getHeaderSize())); + return (T) NativeMemoryTracking.recordMalloc(outerPointer, size, flag); } @Override @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public T realloc(T ptr, UnsignedWord size) { - return libc().realloc(ptr, size); + return realloc(ptr, size, NmtFlag.Default.ordinal()); + } + + @Override + public T realloc(T ptr, UnsignedWord size, int flag) { + Pointer outerPointer = libc().realloc(ptr, size.add(NativeMemoryTracking.getHeaderSize())); + return (T) NativeMemoryTracking.recordMalloc(outerPointer, size, flag); } @Override @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void free(PointerBase ptr) { + NativeMemoryTracking.deaccountMalloc(ptr); + libc().free(((Pointer) ptr).subtract(NativeMemoryTracking.getHeaderSize())); + } + + @Override + public void untrackedFree(PointerBase ptr) { libc().free(ptr); } diff --git a/substratevm/src/com.oracle.svm.core.windows/src/com/oracle/svm/core/windows/WindowsVirtualMemoryProvider.java b/substratevm/src/com.oracle.svm.core.windows/src/com/oracle/svm/core/windows/WindowsVirtualMemoryProvider.java index b833a15f32f7..8ec9f7089b2c 100644 --- a/substratevm/src/com.oracle.svm.core.windows/src/com/oracle/svm/core/windows/WindowsVirtualMemoryProvider.java +++ b/substratevm/src/com.oracle.svm.core.windows/src/com/oracle/svm/core/windows/WindowsVirtualMemoryProvider.java @@ -46,6 +46,9 @@ import com.oracle.svm.core.c.CGlobalDataFactory; import com.oracle.svm.core.c.function.CEntryPointActions; import com.oracle.svm.core.feature.AutomaticallyRegisteredImageSingleton; +import com.oracle.svm.core.nmt.NativeMemoryTracking; +import com.oracle.svm.core.nmt.NmtFlag; +import com.oracle.svm.core.nmt.NmtVirtualMemoryData; import com.oracle.svm.core.os.VirtualMemoryProvider; import com.oracle.svm.core.util.PointerUtils; import com.oracle.svm.core.util.UnsignedUtils; @@ -122,10 +125,13 @@ private static int accessAsProt(int access) { /** Sentinel value indicating that no special alignment is required. */ private static final UnsignedWord UNALIGNED = WordFactory.zero(); - - @Override @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public Pointer reserve(UnsignedWord nbytes, UnsignedWord alignment, boolean executable) { + return reserve(nbytes, alignment, executable, WordFactory.nullPointer()); + } + @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public Pointer reserve(UnsignedWord nbytes, UnsignedWord alignment, boolean executable, NmtVirtualMemoryData nmtData) { if (nbytes.equal(0)) { return WordFactory.nullPointer(); } @@ -160,6 +166,11 @@ public Pointer reserve(UnsignedWord nbytes, UnsignedWord alignment, boolean exec if (reserved.isNull()) { return WordFactory.nullPointer(); } + if (nmtData.isNull()) { + NativeMemoryTracking.recordReserve(nbytes.add(requiredAlignment), NmtFlag.Default.ordinal()); + } else { + nmtData.setReserved(nmtData.getReserved().add(nbytes.add(requiredAlignment))); + } return requiredAlignment.equal(UNALIGNED) ? reserved : PointerUtils.roundUp(reserved, requiredAlignment); } @@ -284,6 +295,11 @@ private interface MEM_ADDRESS_REQUIREMENTS extends PointerBase { @Override @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public Pointer mapFile(PointerBase start, UnsignedWord nbytes, WordBase fileHandle, UnsignedWord offset, int access) { + return mapFile(start, nbytes, fileHandle, offset, access, WordFactory.nullPointer()); + } + @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public Pointer mapFile(PointerBase start, UnsignedWord nbytes, WordBase fileHandle, UnsignedWord offset, int access, NmtVirtualMemoryData nmtData) { if ((start.isNonNull() && !isAligned(start)) || nbytes.equal(0)) { return WordFactory.nullPointer(); } @@ -313,6 +329,11 @@ public Pointer mapFile(PointerBase start, UnsignedWord nbytes, WordBase fileHand /* Restore a normal allocation as the caller is unaware of placeholders. */ replacePlaceholder(start, nbytes); } + if (nmtData.isNull()) { + NativeMemoryTracking.recordCommit(nbytes, NmtFlag.Default.ordinal()); + } else { + nmtData.setCommitted(nmtData.getCommitted().add(nbytes)); + } return fileView; } @@ -352,10 +373,21 @@ Pointer invoke(HANDLE fileMapping, HANDLE process, PointerBase baseAddress, long @Override @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public Pointer commit(PointerBase start, UnsignedWord nbytes, int access) { + return commit(start, nbytes, access, WordFactory.nullPointer()); + } + @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public Pointer commit(PointerBase start, UnsignedWord nbytes, int access, NmtVirtualMemoryData nmtData) { if ((start.isNonNull() && !isAligned(start)) || nbytes.equal(0)) { return WordFactory.nullPointer(); } + if (nmtData.isNull()) { + NativeMemoryTracking.recordCommit(nbytes, NmtFlag.Default.ordinal()); + } else { + nmtData.setCommitted(nmtData.getCommitted().add(nbytes)); + } + /* * VirtualAlloc only guarantees the zeroing for freshly committed pages (i.e., the content * of pages that were already committed earlier won't be touched). diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/Isolates.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/Isolates.java index 1723b3fcba9d..d8a25b843b8c 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/Isolates.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/Isolates.java @@ -39,6 +39,7 @@ import com.oracle.svm.core.c.function.CEntryPointSetup; import com.oracle.svm.core.code.CodeInfoTable; import com.oracle.svm.core.heap.Heap; +import com.oracle.svm.core.nmt.NmtVirtualMemoryData; import com.oracle.svm.core.os.CommittedMemoryProvider; import com.oracle.svm.core.util.VMError; @@ -114,9 +115,9 @@ public static int checkIsolate(Isolate isolate) { } @Uninterruptible(reason = "Thread state not yet set up.") - public static int create(WordPointer isolatePointer, CEntryPointCreateIsolateParameters parameters) { + public static int create(WordPointer isolatePointer, CEntryPointCreateIsolateParameters parameters, NmtVirtualMemoryData nmtData) { WordPointer heapBasePointer = StackValue.get(WordPointer.class); - int result = CommittedMemoryProvider.get().initialize(heapBasePointer, parameters); + int result = CommittedMemoryProvider.get().initialize(heapBasePointer, parameters, nmtData); if (result != CEntryPointErrors.NO_ERROR) { return result; } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/VMInspectionOptions.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/VMInspectionOptions.java index bf1d2a4e5946..90e370c454a6 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/VMInspectionOptions.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/VMInspectionOptions.java @@ -57,11 +57,12 @@ public final class VMInspectionOptions { private static final String MONITORING_JMXCLIENT_NAME = "jmxclient"; private static final String MONITORING_JMXSERVER_NAME = "jmxserver"; private static final String MONITORING_THREADDUMP_NAME = "threaddump"; + private static final String MONITORING_NMT_NAME = "nmt"; private static final List MONITORING_ALL_VALUES = List.of(MONITORING_HEAPDUMP_NAME, MONITORING_JFR_NAME, MONITORING_JVMSTAT_NAME, MONITORING_JMXCLIENT_NAME, MONITORING_JMXSERVER_NAME, - MONITORING_THREADDUMP_NAME, MONITORING_ALL_NAME, MONITORING_DEFAULT_NAME); + MONITORING_THREADDUMP_NAME, MONITORING_NMT_NAME, MONITORING_ALL_NAME, MONITORING_DEFAULT_NAME); private static final String MONITORING_ALLOWED_VALUES_TEXT = "'" + MONITORING_HEAPDUMP_NAME + "', '" + MONITORING_JFR_NAME + "', '" + MONITORING_JVMSTAT_NAME + "', '" + MONITORING_JMXSERVER_NAME + - "' (experimental), '" + MONITORING_JMXCLIENT_NAME + "' (experimental), '" + MONITORING_THREADDUMP_NAME + "', or '" + MONITORING_ALL_NAME + + "' (experimental), '" + MONITORING_JMXCLIENT_NAME + "' (experimental), '" + MONITORING_THREADDUMP_NAME + "', '" + MONITORING_NMT_NAME + "', or '" + MONITORING_ALL_NAME + "' (deprecated behavior: defaults to '" + MONITORING_ALL_NAME + "' if no argument is provided)"; static { @@ -163,6 +164,15 @@ public static boolean hasThreadDumpSupport() { return hasAllOrKeywordMonitoringSupport(MONITORING_THREADDUMP_NAME) || DeprecatedOptions.DumpThreadStacksOnSignal.getValue(); } + @Fold + public static boolean hasNmtSupport() { + return hasAllOrKeywordMonitoringSupport(MONITORING_NMT_NAME) && !Platform.includedIn(WINDOWS.class); // TODO + // no + // windows + // for + // now? + } + @Option(help = "Dumps all runtime compiled methods on SIGUSR2.", type = OptionType.User) // public static final HostedOptionKey DumpRuntimeCompilationOnSignal = new HostedOptionKey<>(false); diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/snippets/CEntryPointSnippets.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/snippets/CEntryPointSnippets.java index 4d11dab84c82..7f6f4dbf0cd1 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/snippets/CEntryPointSnippets.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/snippets/CEntryPointSnippets.java @@ -98,6 +98,10 @@ import com.oracle.svm.core.jdk.PlatformNativeLibrarySupport; import com.oracle.svm.core.jdk.RuntimeSupport; import com.oracle.svm.core.log.Log; +import com.oracle.svm.core.nmt.HasNmtSupport; +import com.oracle.svm.core.nmt.NmtFlag; +import com.oracle.svm.core.nmt.NmtVirtualMemoryData; +import com.oracle.svm.core.nmt.NativeMemoryTracking; import com.oracle.svm.core.option.RuntimeOptionParser; import com.oracle.svm.core.os.MemoryProtectionProvider; import com.oracle.svm.core.os.VirtualMemoryProvider; @@ -243,7 +247,13 @@ private static int createIsolate(CEntryPointCreateIsolateParameters providedPara } WordPointer isolatePtr = StackValue.get(WordPointer.class); - int error = Isolates.create(isolatePtr, parameters); + NmtVirtualMemoryData nmtData = WordFactory.nullPointer(); + if (HasNmtSupport.get()) { + nmtData = StackValue.get(NmtVirtualMemoryData.class); + nmtData.setCommitted(WordFactory.zero()); + nmtData.setReserved(WordFactory.zero()); + } + int error = Isolates.create(isolatePtr, parameters, nmtData); if (error != CEntryPointErrors.NO_ERROR) { return error; } @@ -252,6 +262,11 @@ private static int createIsolate(CEntryPointCreateIsolateParameters providedPara setHeapBase(Isolates.getHeapBase(isolate)); } + if (HasNmtSupport.get()) { + NativeMemoryTracking.recordCommit(nmtData.getCommitted(), NmtFlag.Default.ordinal()); + NativeMemoryTracking.recordReserve(nmtData.getReserved(), NmtFlag.Default.ordinal()); + } + return createIsolate0(parsedArgs, isolate, vmThreadSize); } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jfr/JfrChunkWriter.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jfr/JfrChunkWriter.java index 219bdb4c5b7c..8e742d63000d 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jfr/JfrChunkWriter.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jfr/JfrChunkWriter.java @@ -205,6 +205,7 @@ public void markChunkFinal() { */ public void closeFile() { assert lock.isOwner(); + com.oracle.svm.core.nmt.NativeMemoryTracking.printStats(); /* * Switch to a new epoch. This is done at a safepoint to ensure that we end up with * consistent data, even if multiple threads have JFR events in progress. diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/HasNmtSupport.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/HasNmtSupport.java new file mode 100644 index 000000000000..19607542b0a0 --- /dev/null +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/HasNmtSupport.java @@ -0,0 +1,43 @@ +/* + * Copyright (c) 2023, 2023, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2023, 2023, Red Hat Inc. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +package com.oracle.svm.core.nmt; + +import java.util.function.BooleanSupplier; +import jdk.graal.compiler.api.replacements.Fold; +import org.graalvm.nativeimage.ImageSingletons; + +public class HasNmtSupport implements BooleanSupplier { + @Override + public boolean getAsBoolean() { + return get(); + } + + @Fold + public static boolean get() { + return ImageSingletons.contains(NativeMemoryTracking.class); + } +} diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/MallocHeader.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/MallocHeader.java new file mode 100644 index 000000000000..84c680b35dd8 --- /dev/null +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/MallocHeader.java @@ -0,0 +1,46 @@ +/* + * Copyright (c) 2023, 2023, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2023, 2023, Red Hat Inc. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +package com.oracle.svm.core.nmt; + +import org.graalvm.nativeimage.c.struct.RawField; +import org.graalvm.nativeimage.c.struct.RawStructure; +import org.graalvm.word.PointerBase; + +@RawStructure +public interface MallocHeader extends PointerBase { + @RawField + long getSize(); + + @RawField + void setSize(long value); + + @RawField + int getFlag(); + + @RawField + void setFlag(int value); +} diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/MallocMemoryInfo.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/MallocMemoryInfo.java new file mode 100644 index 000000000000..4393b5b93b2f --- /dev/null +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/MallocMemoryInfo.java @@ -0,0 +1,65 @@ +/* + * Copyright (c) 2023, 2023, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2023, 2023, Red Hat Inc. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +package com.oracle.svm.core.nmt; + +import com.oracle.svm.core.jdk.UninterruptibleUtils.AtomicLong; + +import com.oracle.svm.core.Uninterruptible; +import org.graalvm.word.UnsignedWord; +import org.graalvm.nativeimage.Platforms; +import org.graalvm.nativeimage.Platform; + +class MallocMemoryInfo { + private AtomicLong count; + private AtomicLong size; + + @Platforms(Platform.HOSTED_ONLY.class) + MallocMemoryInfo() { + count = new AtomicLong(0); + size = new AtomicLong(0); + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + void recordMalloc(UnsignedWord allocationSize) { + count.incrementAndGet(); + size.addAndGet(allocationSize.rawValue()); + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + void deaccountMalloc(long allocationSize) { + count.decrementAndGet(); + size.addAndGet(-allocationSize); + } + + long getSize() { + return size.get(); + } + + long getCount() { + return count.get(); + } +} diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/MallocMemorySnapshot.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/MallocMemorySnapshot.java new file mode 100644 index 000000000000..07336bf784fd --- /dev/null +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/MallocMemorySnapshot.java @@ -0,0 +1,56 @@ +/* + * Copyright (c) 2023, 2023, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2023, 2023, Red Hat Inc. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +package com.oracle.svm.core.nmt; + +import com.oracle.svm.core.Uninterruptible; +import org.graalvm.nativeimage.Platforms; +import org.graalvm.nativeimage.Platform; + +public class MallocMemorySnapshot { + private MallocMemoryInfo[] categories; + private MallocMemoryInfo total; + + @Platforms(Platform.HOSTED_ONLY.class) + MallocMemorySnapshot() { + total = new MallocMemoryInfo(); + categories = new MallocMemoryInfo[NmtFlag.values().length]; + for (int i = 0; i < categories.length; i++) { + categories[i] = new MallocMemoryInfo(); + } + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + MallocMemoryInfo getInfoByCategory(int flag) { + assert flag < categories.length; + return categories[flag]; + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public MallocMemoryInfo getTotalInfo() { + return total; + } +} diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/NativeMemoryTracking.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/NativeMemoryTracking.java new file mode 100644 index 000000000000..807185a0725b --- /dev/null +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/NativeMemoryTracking.java @@ -0,0 +1,145 @@ +/* + * Copyright (c) 2023, 2023, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2023, 2023, Red Hat Inc. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +package com.oracle.svm.core.nmt; + +import com.oracle.svm.core.Uninterruptible; +import com.oracle.svm.core.util.UnsignedUtils; +import org.graalvm.nativeimage.c.struct.SizeOf; +import org.graalvm.word.WordFactory; +import com.oracle.svm.core.config.ConfigurationValues; +import org.graalvm.word.Pointer; +import org.graalvm.word.PointerBase; +import org.graalvm.word.UnsignedWord; + +import org.graalvm.nativeimage.Platforms; +import org.graalvm.nativeimage.Platform; + +import org.graalvm.nativeimage.ImageSingletons; + +public class NativeMemoryTracking { + public com.oracle.svm.core.nmt.MallocMemorySnapshot mallocMemorySnapshot; + private VirtualMemorySnapshot virtualMemorySnapshot; + private boolean enabled; + + @Platforms(Platform.HOSTED_ONLY.class) + public NativeMemoryTracking() { + enabled = true; // TODO enable via the same runtime options in hotspot + mallocMemorySnapshot = new MallocMemorySnapshot(); + virtualMemorySnapshot = new VirtualMemorySnapshot(); + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public static void setEnabled(boolean enabled) { + get().enabled = enabled; + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public static UnsignedWord getHeaderSize() { + if (HasNmtSupport.get() && get().enabled) { // *** SubstrateJVM calls HasJfrSupport + return UnsignedUtils.roundUp(SizeOf.unsigned(MallocHeader.class), WordFactory.unsigned(ConfigurationValues.getTarget().wordSize)); + } + return WordFactory.zero(); + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + private static NativeMemoryTracking get() { + return ImageSingletons.lookup(NativeMemoryTracking.class); + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public static Pointer recordMalloc(Pointer outerPointer, UnsignedWord size, int flag) { + if (HasNmtSupport.get() && get().enabled) { + get().mallocMemorySnapshot.getInfoByCategory(flag).recordMalloc(size); + get().mallocMemorySnapshot.getTotalInfo().recordMalloc(size); + return initializeHeader(outerPointer, size, flag); + } + return outerPointer; + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public static void deaccountMalloc(PointerBase ptr) { + if (HasNmtSupport.get() && get().enabled) { + MallocHeader header = (MallocHeader) ((Pointer) ptr).subtract(getHeaderSize()); + get().mallocMemorySnapshot.getInfoByCategory(header.getFlag()).deaccountMalloc(header.getSize()); + get().mallocMemorySnapshot.getTotalInfo().deaccountMalloc(header.getSize()); + } + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + private static Pointer initializeHeader(Pointer outerPointer, UnsignedWord size, int flag) { + MallocHeader mallocHeader = (MallocHeader) outerPointer; + mallocHeader.setSize(size.rawValue()); + mallocHeader.setFlag(flag); + return ((Pointer) mallocHeader).add(NativeMemoryTracking.getHeaderSize()); + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public static void recordReserve(UnsignedWord size, int flag) { + if (HasNmtSupport.get() && get().enabled) { + get().virtualMemorySnapshot.getInfoByCategory(flag).recordReserved(size); + get().virtualMemorySnapshot.getTotalInfo().recordReserved(size); + } + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public static void recordCommit(UnsignedWord size, int flag) { + if (HasNmtSupport.get() && get().enabled) { + get().virtualMemorySnapshot.getInfoByCategory(flag).recordCommitted(size); + get().virtualMemorySnapshot.getTotalInfo().recordCommitted(size); + } + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public static void recordUncommit(UnsignedWord size, int flag) { + if (HasNmtSupport.get() && get().enabled) { + get().virtualMemorySnapshot.getInfoByCategory(flag).recordUncommit(size); + get().virtualMemorySnapshot.getTotalInfo().recordUncommit(size); + } + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public static void recordFree(UnsignedWord size, int flag) { + if (HasNmtSupport.get() && get().enabled) { + get().virtualMemorySnapshot.getInfoByCategory(flag).recordFree(size); + get().virtualMemorySnapshot.getTotalInfo().recordFree(size); + } + } + + public static void printStats() { + if (HasNmtSupport.get()) { + System.out.println("MALLOC"); + for (int i = 0; i < com.oracle.svm.core.nmt.NmtFlag.values().length; i++) { + + } + System.out.println("total size:" + get().mallocMemorySnapshot.getTotalInfo().getSize()); + System.out.println("total count:" + get().mallocMemorySnapshot.getTotalInfo().getCount()); + System.out.println("Virtual"); + System.out.println("total reserved:" + get().virtualMemorySnapshot.getTotalInfo().getReservedSize()); + System.out.println("total committed:" + get().virtualMemorySnapshot.getTotalInfo().getCommittedSize()); + } + } +} diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/NmtFeature.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/NmtFeature.java new file mode 100644 index 000000000000..6b442f9be95d --- /dev/null +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/NmtFeature.java @@ -0,0 +1,46 @@ +/* + * Copyright (c) 2023, 2023, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2023, 2023, Red Hat Inc. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +package com.oracle.svm.core.nmt; + +import com.oracle.svm.core.VMInspectionOptions; +import org.graalvm.nativeimage.ImageSingletons; + +import com.oracle.svm.core.feature.AutomaticallyRegisteredFeature; +import com.oracle.svm.core.feature.InternalFeature; + + +@AutomaticallyRegisteredFeature +public class NmtFeature implements InternalFeature { + @Override + public boolean isInConfiguration(IsInConfigurationAccess access) { + return VMInspectionOptions.hasNmtSupport(); + } + @Override + public void afterRegistration(AfterRegistrationAccess access) { + ImageSingletons.add(NativeMemoryTracking.class, new NativeMemoryTracking()); + } +} diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/NmtFlag.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/NmtFlag.java new file mode 100644 index 000000000000..0b6059934081 --- /dev/null +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/NmtFlag.java @@ -0,0 +1,45 @@ +/* + * Copyright (c) 2023, 2023, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2023, 2023, Red Hat Inc. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package com.oracle.svm.core.nmt; + +import com.oracle.svm.core.Uninterruptible; + +public enum NmtFlag { + Jfr("jfr"), + Nmt("native-memory-tracking"), + Default("untracked"); + + private final String name; + + NmtFlag(String name) { + this.name = name; + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public String getName() { + return name; + } +} diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/NmtVirtualMemoryData.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/NmtVirtualMemoryData.java new file mode 100644 index 000000000000..2d98a1c80725 --- /dev/null +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/NmtVirtualMemoryData.java @@ -0,0 +1,47 @@ +/* + * Copyright (c) 2023, 2023, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2023, 2023, Red Hat Inc. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +package com.oracle.svm.core.nmt; + +import org.graalvm.nativeimage.c.struct.RawField; +import org.graalvm.nativeimage.c.struct.RawStructure; +import org.graalvm.word.PointerBase; +import org.graalvm.word.UnsignedWord; + +@RawStructure +public interface NmtVirtualMemoryData extends PointerBase { + @RawField + UnsignedWord getReserved(); + + @RawField + void setReserved(UnsignedWord value); + + @RawField + UnsignedWord getCommitted(); + + @RawField + void setCommitted(UnsignedWord value); +} diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/VirtualMemoryInfo.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/VirtualMemoryInfo.java new file mode 100644 index 000000000000..9ac7e07d5e30 --- /dev/null +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/VirtualMemoryInfo.java @@ -0,0 +1,74 @@ +/* + * Copyright (c) 2023, 2023, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2023, 2023, Red Hat Inc. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +package com.oracle.svm.core.nmt; + +import com.oracle.svm.core.Uninterruptible; +import com.oracle.svm.core.jdk.UninterruptibleUtils.AtomicLong; +import org.graalvm.nativeimage.Platform; +import org.graalvm.nativeimage.Platforms; +import org.graalvm.word.UnsignedWord; + +class VirtualMemoryInfo { // TODO track peak + private AtomicLong reservedSize; + private AtomicLong committedSize; + + @Platforms(Platform.HOSTED_ONLY.class) + VirtualMemoryInfo() { + reservedSize = new AtomicLong(0); + committedSize = new AtomicLong(0); + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + void recordReserved(UnsignedWord size) { + reservedSize.addAndGet(size.rawValue()); + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + void recordCommitted(UnsignedWord size) { // *** hotspot doesnt adjust reserved when mem is + // committed. The same block is counted as both + // reserved and committed. + committedSize.addAndGet(size.rawValue()); + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + void recordUncommit(UnsignedWord size) { + committedSize.addAndGet(-size.rawValue()); + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + void recordFree(UnsignedWord size) { + reservedSize.addAndGet(-size.rawValue()); + } + + long getReservedSize() { + return reservedSize.get(); + } + + long getCommittedSize() { + return committedSize.get(); + } +} diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/VirtualMemorySnapshot.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/VirtualMemorySnapshot.java new file mode 100644 index 000000000000..59e8b17a0ad8 --- /dev/null +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/VirtualMemorySnapshot.java @@ -0,0 +1,58 @@ +/* + * Copyright (c) 2023, 2023, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2023, 2023, Red Hat Inc. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +package com.oracle.svm.core.nmt; + +import com.oracle.svm.core.Uninterruptible; +import org.graalvm.nativeimage.Platform; +import org.graalvm.nativeimage.Platforms; + +class VirtualMemorySnapshot { // TODO this is the same as MallocMemorySnapshot. Maybe we can reduce + // code duplication. Lets do that last once we know ALL the methods + // these classes we'll need + private VirtualMemoryInfo[] categories; + private VirtualMemoryInfo total; + + @Platforms(Platform.HOSTED_ONLY.class) + VirtualMemorySnapshot() { + total = new VirtualMemoryInfo(); + categories = new VirtualMemoryInfo[NmtFlag.values().length]; + for (int i = 0; i < categories.length; i++) { + categories[i] = new VirtualMemoryInfo(); + } + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + VirtualMemoryInfo getInfoByCategory(int flag) { + assert flag < categories.length; + return categories[flag]; + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + VirtualMemoryInfo getTotalInfo() { + return total; + } +} diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/os/AbstractCopyingImageHeapProvider.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/os/AbstractCopyingImageHeapProvider.java index afe8fde670ff..c1c5399dc95e 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/os/AbstractCopyingImageHeapProvider.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/os/AbstractCopyingImageHeapProvider.java @@ -40,6 +40,7 @@ import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.c.function.CEntryPointErrors; import com.oracle.svm.core.heap.Heap; +import com.oracle.svm.core.nmt.NmtVirtualMemoryData; import com.oracle.svm.core.os.VirtualMemoryProvider.Access; import com.oracle.svm.core.util.UnsignedUtils; @@ -51,7 +52,7 @@ public boolean guaranteesHeapPreferredAddressSpaceAlignment() { @Override @Uninterruptible(reason = "Called during isolate initialization.") - public int initialize(Pointer reservedAddressSpace, UnsignedWord reservedSize, WordPointer basePointer, WordPointer endPointer) { + public int initialize(Pointer reservedAddressSpace, UnsignedWord reservedSize, WordPointer basePointer, WordPointer endPointer, NmtVirtualMemoryData nmtData) { boolean haveDynamicMethodResolution = DynamicMethodAddressResolutionHeapSupport.isEnabled(); UnsignedWord preHeapRequiredBytes = WordFactory.zero(); UnsignedWord alignment = WordFactory.unsigned(Heap.getHeap().getPreferredAddressSpaceAlignment()); @@ -71,7 +72,7 @@ public int initialize(Pointer reservedAddressSpace, UnsignedWord reservedSize, W Pointer heapBase; Pointer allocatedMemory = WordFactory.nullPointer(); if (reservedAddressSpace.isNull()) { - heapBase = allocatedMemory = VirtualMemoryProvider.get().reserve(totalAddressSpaceSize, alignment, false); + heapBase = allocatedMemory = VirtualMemoryProvider.get().reserve(totalAddressSpaceSize, alignment, false, nmtData); if (allocatedMemory.isNull()) { return CEntryPointErrors.RESERVE_ADDRESS_SPACE_FAILED; } @@ -100,7 +101,7 @@ public int initialize(Pointer reservedAddressSpace, UnsignedWord reservedSize, W Word imageHeapBegin = IMAGE_HEAP_BEGIN.get(); UnsignedWord imageHeapSizeInFile = getImageHeapSizeInFile(); Pointer imageHeap = heapBase.add(Heap.getHeap().getImageHeapOffsetInAddressSpace()); - int result = commitAndCopyMemory(imageHeapBegin, imageHeapSizeInFile, imageHeap); + int result = commitAndCopyMemory(imageHeapBegin, imageHeapSizeInFile, imageHeap, nmtData); if (result != CEntryPointErrors.NO_ERROR) { freeImageHeap(allocatedMemory); return result; @@ -145,8 +146,8 @@ public int initialize(Pointer reservedAddressSpace, UnsignedWord reservedSize, W } @Uninterruptible(reason = "Called during isolate initialization.") - protected int commitAndCopyMemory(Pointer loadedImageHeap, UnsignedWord imageHeapSize, Pointer newImageHeap) { - Pointer actualNewImageHeap = VirtualMemoryProvider.get().commit(newImageHeap, imageHeapSize, Access.READ | Access.WRITE); + protected int commitAndCopyMemory(Pointer loadedImageHeap, UnsignedWord imageHeapSize, Pointer newImageHeap, NmtVirtualMemoryData nmtData) { + Pointer actualNewImageHeap = VirtualMemoryProvider.get().commit(newImageHeap, imageHeapSize, Access.READ | Access.WRITE, nmtData); if (actualNewImageHeap.isNull() || actualNewImageHeap.notEqual(newImageHeap)) { return CEntryPointErrors.RESERVE_ADDRESS_SPACE_FAILED; } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/os/CommittedMemoryProvider.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/os/CommittedMemoryProvider.java index c1111a02654c..802de4a1841e 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/os/CommittedMemoryProvider.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/os/CommittedMemoryProvider.java @@ -36,6 +36,7 @@ import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.c.function.CEntryPointCreateIsolateParameters; import com.oracle.svm.core.heap.Heap; +import com.oracle.svm.core.nmt.NmtVirtualMemoryData; /** * A provider of ranges of committed memory, which is virtual memory that is backed by physical @@ -62,7 +63,7 @@ static CommittedMemoryProvider get() { * @return zero in case of success, non-zero in case of an error. */ @Uninterruptible(reason = "Still being initialized.") - int initialize(WordPointer heapBasePointer, CEntryPointCreateIsolateParameters parameters); + int initialize(WordPointer heapBasePointer, CEntryPointCreateIsolateParameters parameters, NmtVirtualMemoryData nmtData); /** * Tear down for the current isolate. This must be the last method of this interface diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/os/ImageHeapProvider.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/os/ImageHeapProvider.java index b277db95d154..2a4dc8b78e6a 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/os/ImageHeapProvider.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/os/ImageHeapProvider.java @@ -33,6 +33,7 @@ import com.oracle.svm.core.c.function.CEntryPointErrors; import com.oracle.svm.core.heap.Heap; +import com.oracle.svm.core.nmt.NmtVirtualMemoryData; /** * Provides new instances of the image heap for creating isolates. The same image heap provider @@ -79,7 +80,7 @@ static ImageHeapProvider get() { * written. May be null if this value is not required. * @return a result code from {@link CEntryPointErrors}. */ - int initialize(Pointer reservedAddressSpace, UnsignedWord reservedSize, WordPointer basePointer, WordPointer endPointer); + int initialize(Pointer reservedAddressSpace, UnsignedWord reservedSize, WordPointer basePointer, WordPointer endPointer, NmtVirtualMemoryData nmtData); /** * Disposes an instance of the image heap that was created with this provider. This method must diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/os/OSCommittedMemoryProvider.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/os/OSCommittedMemoryProvider.java index b09b8bddcd04..14cb24ae1cde 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/os/OSCommittedMemoryProvider.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/os/OSCommittedMemoryProvider.java @@ -41,6 +41,7 @@ import com.oracle.svm.core.c.function.CEntryPointErrors; import com.oracle.svm.core.feature.AutomaticallyRegisteredFeature; import com.oracle.svm.core.feature.InternalFeature; +import com.oracle.svm.core.nmt.NmtVirtualMemoryData; public class OSCommittedMemoryProvider extends AbstractCommittedMemoryProvider { @Platforms(Platform.HOSTED_ONLY.class) @@ -49,7 +50,7 @@ public OSCommittedMemoryProvider() { @Override @Uninterruptible(reason = "Still being initialized.") - public int initialize(WordPointer heapBasePointer, CEntryPointCreateIsolateParameters parameters) { + public int initialize(WordPointer heapBasePointer, CEntryPointCreateIsolateParameters parameters, NmtVirtualMemoryData nmtData) { if (!SubstrateOptions.SpawnIsolates.getValue()) { int result = protectSingleIsolateImageHeap(); if (result == CEntryPointErrors.NO_ERROR) { @@ -57,7 +58,7 @@ public int initialize(WordPointer heapBasePointer, CEntryPointCreateIsolateParam } return result; } - return ImageHeapProvider.get().initialize(nullPointer(), zero(), heapBasePointer, nullPointer()); + return ImageHeapProvider.get().initialize(nullPointer(), zero(), heapBasePointer, nullPointer(), nmtData); } @Override diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/os/VirtualMemoryProvider.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/os/VirtualMemoryProvider.java index 7aa211c1bd41..b30edbda9583 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/os/VirtualMemoryProvider.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/os/VirtualMemoryProvider.java @@ -31,6 +31,7 @@ import org.graalvm.word.UnsignedWord; import org.graalvm.word.WordBase; import org.graalvm.word.WordFactory; +import com.oracle.svm.core.nmt.NmtVirtualMemoryData; /** * Primitive operations for low-level virtual memory management. @@ -93,6 +94,8 @@ default UnsignedWord getAlignment() { * address range, or {@link WordFactory#nullPointer()} in case of an error. */ Pointer reserve(UnsignedWord nbytes, UnsignedWord alignment, boolean code); + Pointer reserve(UnsignedWord nbytes, UnsignedWord alignment, boolean code, NmtVirtualMemoryData nmtData); + /** * Map a region of an open file to the specified address range. When {@linkplain Access#WRITE @@ -115,6 +118,8 @@ default UnsignedWord getAlignment() { * of an error. */ Pointer mapFile(PointerBase start, UnsignedWord nbytes, WordBase fileHandle, UnsignedWord offset, int access); + Pointer mapFile(PointerBase start, UnsignedWord nbytes, WordBase fileHandle, UnsignedWord offset, int access, NmtVirtualMemoryData nmtData); + /** * Commit an address range so that physical memory or swap memory can be provisioned for it, and @@ -143,6 +148,7 @@ default UnsignedWord getAlignment() { * case of an error, such as inadequate physical memory. */ Pointer commit(PointerBase start, UnsignedWord nbytes, int access); + Pointer commit(PointerBase start, UnsignedWord nbytes, int access, NmtVirtualMemoryData nmtData); /** * Change the protection of a committed address range, or of a subrange of a committed address diff --git a/substratevm/src/com.oracle.svm.test/src/META-INF/native-image/com.oracle.svm.test/native-image.properties b/substratevm/src/com.oracle.svm.test/src/META-INF/native-image/com.oracle.svm.test/native-image.properties index 095bac6acbb2..e1470477f070 100644 --- a/substratevm/src/com.oracle.svm.test/src/META-INF/native-image/com.oracle.svm.test/native-image.properties +++ b/substratevm/src/com.oracle.svm.test/src/META-INF/native-image/com.oracle.svm.test/native-image.properties @@ -8,5 +8,5 @@ Args= \ --features=com.oracle.svm.test.jfr.JfrTestFeature \ --add-opens=java.base/java.lang=ALL-UNNAMED \ --add-exports=org.graalvm.nativeimage.base/com.oracle.svm.util=ALL-UNNAMED \ - --enable-monitoring=jvmstat,jfr,jmxserver,jmxclient \ + --enable-monitoring=jvmstat,jfr,jmxserver,jmxclient,nmt \ -J--enable-preview \ No newline at end of file From 58cfa1d5be13b597edc33d5dfaaca67222c8e6c6 Mon Sep 17 00:00:00 2001 From: Robert Toyonaga Date: Thu, 9 Nov 2023 16:08:01 -0500 Subject: [PATCH 3/7] Malloc headers. Labels. Clean up. wip handle preinit clean up and add some manual labels minor cleanup and fixes Label call sites and clean up mtThread Remove virtual memory tracking code wip remove more vmem tracking code comments and refactor more small fixes, refactore, comments comment unchecked cast label Unsafe callsite. Clean up. javadoc --- .../graalvm/nativeimage/UnmanagedMemory.java | 21 +- .../impl/UnmanagedMemorySupport.java | 2 +- .../posix/PosixVirtualMemoryProvider.java | 61 +-- .../posix/UnmanagedMemorySupportImpl.java | 46 +- .../posix/linux/LinuxImageHeapProvider.java | 19 +- .../windows/WindowsImageHeapProvider.java | 9 +- .../WindowsUnmanagedMemorySupportImpl.java | 21 +- .../windows/WindowsVirtualMemoryProvider.java | 36 +- .../src/com/oracle/svm/core/Isolates.java | 5 +- .../com/oracle/svm/core/SubstrateOptions.java | 3 + .../oracle/svm/core/VMInspectionOptions.java | 6 +- .../AbstractUninterruptibleHashtable.java | 22 +- .../graal/snippets/CEntryPointSnippets.java | 17 +- .../svm/core/jdk/SunMiscSubstitutions.java | 5 +- .../oracle/svm/core/jfr/JfrBufferAccess.java | 3 +- .../svm/core/jfr/JfrBufferNodeAccess.java | 3 +- .../oracle/svm/core/jfr/JfrChunkWriter.java | 1 - .../svm/core/jfr/JfrStackTraceRepository.java | 3 +- .../com/oracle/svm/core/nmt/MallocHeader.java | 5 + .../oracle/svm/core/nmt/MallocMemoryInfo.java | 5 +- .../svm/core/nmt/NativeMemoryTracking.java | 450 ++++++++++++++++-- .../com/oracle/svm/core/nmt/NmtFeature.java | 16 +- .../src/com/oracle/svm/core/nmt/NmtFlag.java | 16 +- ...emorySnapshot.java => NmtTestFeature.java} | 35 +- .../com/oracle/svm/core/nmt/PreInitTable.java | 102 ++++ ...tualMemoryData.java => ReturnAddress.java} | 15 +- .../svm/core/nmt/VirtualMemoryInfo.java | 74 --- .../os/AbstractCopyingImageHeapProvider.java | 11 +- .../svm/core/os/CommittedMemoryProvider.java | 3 +- .../oracle/svm/core/os/ImageHeapProvider.java | 3 +- .../core/os/OSCommittedMemoryProvider.java | 5 +- .../svm/core/os/VirtualMemoryProvider.java | 6 - .../com/oracle/svm/core/thread/VMThreads.java | 3 +- .../native-image.properties | 3 +- .../com/oracle/svm/test/nmt/TestBasic.java | 122 +++++ 35 files changed, 807 insertions(+), 350 deletions(-) rename substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/{VirtualMemorySnapshot.java => NmtTestFeature.java} (55%) create mode 100644 substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/PreInitTable.java rename substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/{NmtVirtualMemoryData.java => ReturnAddress.java} (84%) delete mode 100644 substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/VirtualMemoryInfo.java create mode 100644 substratevm/src/com.oracle.svm.test/src/com/oracle/svm/test/nmt/TestBasic.java diff --git a/sdk/src/org.graalvm.nativeimage/src/org/graalvm/nativeimage/UnmanagedMemory.java b/sdk/src/org.graalvm.nativeimage/src/org/graalvm/nativeimage/UnmanagedMemory.java index bcc7f962e262..09a3509e8c0d 100644 --- a/sdk/src/org.graalvm.nativeimage/src/org/graalvm/nativeimage/UnmanagedMemory.java +++ b/sdk/src/org.graalvm.nativeimage/src/org/graalvm/nativeimage/UnmanagedMemory.java @@ -76,6 +76,14 @@ public static T malloc(UnsignedWord size) { return result; } + public static T malloc(UnsignedWord size, int flag) { + T result = ImageSingletons.lookup(UnmanagedMemorySupport.class).malloc(size, flag); + if (result.isNull()) { + throw new OutOfMemoryError("malloc of unmanaged memory"); + } + return result; + } + /** * Allocates {@code size} bytes of unmanaged memory. The content of the memory is undefined. *

@@ -138,6 +146,14 @@ public static T realloc(T ptr, UnsignedWord size) { return result; } + public static T realloc(T ptr, UnsignedWord size, int flag) { + T result = ImageSingletons.lookup(UnmanagedMemorySupport.class).realloc(ptr, size, flag); + if (result.isNull()) { + throw new OutOfMemoryError("realloc of unmanaged memory"); + } + return result; + } + /** * Frees unmanaged memory that was previously allocated using methods of this class. * @@ -148,9 +164,8 @@ public static void free(PointerBase ptr) { } /** - * Temporarily the same as {@link UnmanagedMemory#free(PointerBase)}. Will later be different - * because it will not attempt to perform any NMT operations. This is crucial for releasing - * memory allocated by C libraries which will not have NMT "malloc headers". If + * Will not attempt to perform any NMT operations. This is crucial for releasing memory + * allocated by C libraries which will not have NMT "malloc headers". If * {@link UnmanagedMemory#free(PointerBase)} is used instead, a segfault will occur. */ public static void untrackedFree(PointerBase ptr) { diff --git a/sdk/src/org.graalvm.nativeimage/src/org/graalvm/nativeimage/impl/UnmanagedMemorySupport.java b/sdk/src/org.graalvm.nativeimage/src/org/graalvm/nativeimage/impl/UnmanagedMemorySupport.java index 964ac7b912ac..3bd299b7118c 100644 --- a/sdk/src/org.graalvm.nativeimage/src/org/graalvm/nativeimage/impl/UnmanagedMemorySupport.java +++ b/sdk/src/org.graalvm.nativeimage/src/org/graalvm/nativeimage/impl/UnmanagedMemorySupport.java @@ -49,7 +49,7 @@ public interface UnmanagedMemorySupport { T malloc(UnsignedWord size, int flag); T calloc(UnsignedWord size); - T calloc(UnsignedWord size, int flag); + T calloc(UnsignedWord size, int flag); T realloc(T ptr, UnsignedWord size); T realloc(T ptr, UnsignedWord size, int flag); diff --git a/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/PosixVirtualMemoryProvider.java b/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/PosixVirtualMemoryProvider.java index 1b5868a0244a..cffcbbb4550d 100644 --- a/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/PosixVirtualMemoryProvider.java +++ b/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/PosixVirtualMemoryProvider.java @@ -54,9 +54,6 @@ import com.oracle.svm.core.c.CGlobalDataFactory; import com.oracle.svm.core.feature.AutomaticallyRegisteredFeature; import com.oracle.svm.core.feature.InternalFeature; -import com.oracle.svm.core.nmt.NativeMemoryTracking; -import com.oracle.svm.core.nmt.NmtFlag; -import com.oracle.svm.core.nmt.NmtVirtualMemoryData; import com.oracle.svm.core.os.VirtualMemoryProvider; import com.oracle.svm.core.posix.headers.Unistd; import com.oracle.svm.core.util.PointerUtils; @@ -113,11 +110,6 @@ public UnsignedWord getGranularity() { @Override @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public Pointer reserve(UnsignedWord nbytes, UnsignedWord alignment, boolean executable) { - return reserve(nbytes, alignment, executable, WordFactory.nullPointer()); - } - - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - public Pointer reserve(UnsignedWord nbytes, UnsignedWord alignment, boolean executable, NmtVirtualMemoryData nmtData) { if (nbytes.equal(0)) { return WordFactory.nullPointer(); } @@ -136,32 +128,18 @@ public Pointer reserve(UnsignedWord nbytes, UnsignedWord alignment, boolean exec return nullPointer(); } if (!customAlignment) { - if (nmtData.isNull()) { - NativeMemoryTracking.recordReserve(mappingSize, NmtFlag.Default.ordinal()); - } else { - nmtData.setReserved(nmtData.getReserved().add(mappingSize)); - } return mappingBegin; } - UnsignedWord unmappedSize = WordFactory.zero(); Pointer begin = PointerUtils.roundUp(mappingBegin, alignment); UnsignedWord clippedBegin = begin.subtract(mappingBegin); if (clippedBegin.aboveOrEqual(granularity)) { - UnsignedWord unmapSize = UnsignedUtils.roundDown(clippedBegin, granularity); - munmap(mappingBegin, unmapSize); - unmappedSize.add(unmapSize); + munmap(mappingBegin, UnsignedUtils.roundDown(clippedBegin, granularity)); } Pointer mappingEnd = mappingBegin.add(mappingSize); UnsignedWord clippedEnd = mappingEnd.subtract(begin.add(nbytes)); if (clippedEnd.aboveOrEqual(granularity)) { UnsignedWord rounded = UnsignedUtils.roundDown(clippedEnd, granularity); munmap(mappingEnd.subtract(rounded), rounded); - unmappedSize.add(rounded); - } - if (nmtData.isNull()) { - NativeMemoryTracking.recordReserve(mappingSize.subtract(unmappedSize), NmtFlag.Default.ordinal()); - } else { - nmtData.setReserved(nmtData.getReserved().add(mappingSize.subtract(unmappedSize))); } return begin; } @@ -169,12 +147,6 @@ public Pointer reserve(UnsignedWord nbytes, UnsignedWord alignment, boolean exec @Override @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public Pointer mapFile(PointerBase start, UnsignedWord nbytes, WordBase fileHandle, UnsignedWord offset, int access) { - return mapFile(start, nbytes, fileHandle, offset, access, WordFactory.nullPointer()); - } - - @Override - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - public Pointer mapFile(PointerBase start, UnsignedWord nbytes, WordBase fileHandle, UnsignedWord offset, int access, NmtVirtualMemoryData nmtData) { if ((start.isNonNull() && !isAligned(start)) || nbytes.equal(0)) { return WordFactory.nullPointer(); } @@ -185,28 +157,12 @@ public Pointer mapFile(PointerBase start, UnsignedWord nbytes, WordBase fileHand } int fd = (int) fileHandle.rawValue(); Pointer result = mmap(start, nbytes, accessAsProt(access), flags, fd, offset.rawValue()); - if (result.notEqual(MAP_FAILED())) { - if (nmtData.isNull()) { - // TODO is this also a reserve? [No, the mem has already been reserved in a previous - // call.] - NativeMemoryTracking.recordCommit(nbytes, NmtFlag.Default.ordinal()); - } else { - nmtData.setCommitted(nmtData.getCommitted().add(nbytes)); - } - return result; - } - return WordFactory.nullPointer(); + return result.notEqual(MAP_FAILED()) ? result : WordFactory.nullPointer(); } @Override @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public Pointer commit(PointerBase start, UnsignedWord nbytes, int access) { - return commit(start, nbytes, access, WordFactory.nullPointer()); - } - - @Override - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - public Pointer commit(PointerBase start, UnsignedWord nbytes, int access, NmtVirtualMemoryData nmtData) { if ((start.isNonNull() && !isAligned(start)) || nbytes.equal(0)) { return WordFactory.nullPointer(); } @@ -221,15 +177,7 @@ public Pointer commit(PointerBase start, UnsignedWord nbytes, int access, NmtVir } /* The memory returned by mmap is guaranteed to be zeroed. */ final Pointer result = mmap(start, nbytes, accessAsProt(access), flags, NO_FD, NO_FD_OFFSET); - if (result.notEqual(MAP_FAILED())) { - if (nmtData.isNull()) { - NativeMemoryTracking.recordCommit(nbytes, NmtFlag.Default.ordinal()); - } else { - nmtData.setCommitted(nmtData.getCommitted().add(nbytes)); - } - return result; - } - return nullPointer(); + return result.notEqual(MAP_FAILED()) ? result : nullPointer(); } @Override @@ -248,7 +196,7 @@ public int uncommit(PointerBase start, UnsignedWord nbytes) { if (start.isNull() || !isAligned(start) || nbytes.equal(0)) { return -1; } - NativeMemoryTracking.recordUncommit(nbytes, NmtFlag.Default.ordinal()); + final Pointer result = mmap(start, nbytes, PROT_NONE(), MAP_FIXED() | MAP_ANON() | MAP_PRIVATE() | MAP_NORESERVE(), NO_FD, NO_FD_OFFSET); return result.notEqual(MAP_FAILED()) ? 0 : -1; } @@ -263,7 +211,6 @@ public int free(PointerBase start, UnsignedWord nbytes) { UnsignedWord granularity = getGranularity(); Pointer mappingBegin = PointerUtils.roundDown(start, granularity); UnsignedWord mappingSize = UnsignedUtils.roundUp(nbytes, granularity); - NativeMemoryTracking.recordFree(nbytes, NmtFlag.Default.ordinal()); return munmap(mappingBegin, mappingSize); } diff --git a/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/UnmanagedMemorySupportImpl.java b/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/UnmanagedMemorySupportImpl.java index b5cc431af6a8..8f03ad222df9 100644 --- a/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/UnmanagedMemorySupportImpl.java +++ b/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/UnmanagedMemorySupportImpl.java @@ -32,24 +32,31 @@ import org.graalvm.word.Pointer; import org.graalvm.word.UnsignedWord; import org.graalvm.word.WordFactory; +import org.graalvm.nativeimage.StackValue; import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.feature.AutomaticallyRegisteredImageSingleton; import com.oracle.svm.core.headers.LibCSupport; import com.oracle.svm.core.nmt.NmtFlag; import com.oracle.svm.core.nmt.NativeMemoryTracking; +import com.oracle.svm.core.nmt.ReturnAddress; @AutomaticallyRegisteredImageSingleton(UnmanagedMemorySupport.class) class UnmanagedMemorySupportImpl implements UnmanagedMemorySupport { @Override @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public T malloc(UnsignedWord size) { - return malloc(size, NmtFlag.Default.ordinal()); + return malloc(size, NmtFlag.mtNone.ordinal()); } @Override + @SuppressWarnings("unchecked") @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public T malloc(UnsignedWord size, int flag) { + ReturnAddress ra = StackValue.get(ReturnAddress.class); + if (NativeMemoryTracking.handlePreInitMallocs(size, flag, ra)) { + return (T) ra.get(); + } Pointer outerPointer = libc().malloc(size.add(NativeMemoryTracking.getHeaderSize())); return (T) NativeMemoryTracking.recordMalloc(outerPointer, size, flag); } @@ -57,12 +64,17 @@ public T malloc(UnsignedWord size, int flag) { @Override @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public T calloc(UnsignedWord size) { - return calloc(size, NmtFlag.Default.ordinal()); + return calloc(size, NmtFlag.mtNone.ordinal()); } @Override + @SuppressWarnings("unchecked") @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public T calloc(UnsignedWord size, int flag) { + ReturnAddress ra = StackValue.get(ReturnAddress.class); + if (NativeMemoryTracking.handlePreInitCallocs(size, flag, ra)) { + return (T) ra.get(); + } Pointer outerPointer = libc().calloc(WordFactory.unsigned(1), size.add(NativeMemoryTracking.getHeaderSize())); return (T) NativeMemoryTracking.recordMalloc(outerPointer, size, flag); } @@ -70,21 +82,43 @@ public T calloc(UnsignedWord size, int flag) { @Override @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public T realloc(T ptr, UnsignedWord size) { - return realloc(ptr, size, NmtFlag.Default.ordinal()); + return realloc(ptr, size, NmtFlag.mtNone.ordinal()); } @Override + @SuppressWarnings("unchecked") @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public T realloc(T ptr, UnsignedWord size, int flag) { - Pointer outerPointer = libc().realloc(ptr, size.add(NativeMemoryTracking.getHeaderSize())); - return (T) NativeMemoryTracking.recordMalloc(outerPointer, size, flag); + + ReturnAddress ra = StackValue.get(ReturnAddress.class); + if (NativeMemoryTracking.handlePreInitReallocs((Pointer) ptr, size, flag, ra)) { + return (T) ra.get(); + } + + // Retrieve necessary data from the old block + Pointer oldOuterPointer = ((Pointer) ptr).subtract(NativeMemoryTracking.getHeaderSize()); + long oldSize = NativeMemoryTracking.getAllocationSize(oldOuterPointer); + int oldCategory = NativeMemoryTracking.getAllocationCategory(oldOuterPointer); + + // Perform the realloc + Pointer newOuterPointer = libc().realloc(oldOuterPointer, size.add(NativeMemoryTracking.getHeaderSize())); + + // Only deaccount the old block, if we were successful. + if (newOuterPointer.isNonNull()) { + NativeMemoryTracking.deaccountMalloc(oldSize, oldCategory); + } + + // Account the new block and overwrite the header with the new tracking data + return (T) NativeMemoryTracking.recordMalloc(newOuterPointer, size, flag); } @Override @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void free(PointerBase ptr) { + if (NativeMemoryTracking.handlePreInitFrees((Pointer) ptr)) { + return; + } NativeMemoryTracking.deaccountMalloc(ptr); - // *** tried confirming by changing this. It fails, as expected libc().free(((Pointer) ptr).subtract(NativeMemoryTracking.getHeaderSize())); } diff --git a/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/linux/LinuxImageHeapProvider.java b/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/linux/LinuxImageHeapProvider.java index ce1f84f817cf..555a381ff406 100644 --- a/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/linux/LinuxImageHeapProvider.java +++ b/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/linux/LinuxImageHeapProvider.java @@ -58,7 +58,6 @@ import com.oracle.svm.core.config.ConfigurationValues; import com.oracle.svm.core.headers.LibC; import com.oracle.svm.core.heap.Heap; -import com.oracle.svm.core.nmt.NmtVirtualMemoryData; import com.oracle.svm.core.os.AbstractImageHeapProvider; import com.oracle.svm.core.os.CopyingImageHeapProvider; import com.oracle.svm.core.os.VirtualMemoryProvider; @@ -105,7 +104,7 @@ public boolean guaranteesHeapPreferredAddressSpaceAlignment() { @Override @Uninterruptible(reason = "Called during isolate initialization.") - public int initialize(Pointer reservedAddressSpace, UnsignedWord reservedSize, WordPointer basePointer, WordPointer endPointer, NmtVirtualMemoryData nmtData) { + public int initialize(Pointer reservedAddressSpace, UnsignedWord reservedSize, WordPointer basePointer, WordPointer endPointer) { // If we are the first isolate, we might be able to use the existing image heap (see below) SignedWord fd = CACHED_IMAGE_FD.get().read(); boolean firstIsolate = false; @@ -141,7 +140,7 @@ public int initialize(Pointer reservedAddressSpace, UnsignedWord reservedSize, W // If we cannot find or open the image file, fall back to copy it from memory. if (fd.equal(CANNOT_OPEN_FD)) { - return fallbackCopyingProvider.initialize(reservedAddressSpace, reservedSize, basePointer, endPointer, nmtData); + return fallbackCopyingProvider.initialize(reservedAddressSpace, reservedSize, basePointer, endPointer); } boolean haveDynamicMethodResolution = DynamicMethodAddressResolutionHeapSupport.isEnabled(); @@ -199,7 +198,7 @@ public int initialize(Pointer reservedAddressSpace, UnsignedWord reservedSize, W Pointer heapBase; Pointer allocatedMemory = WordFactory.nullPointer(); if (reservedAddressSpace.isNull()) { - heapBase = allocatedMemory = VirtualMemoryProvider.get().reserve(totalAddressSpaceSize, alignment, false, nmtData); + heapBase = allocatedMemory = VirtualMemoryProvider.get().reserve(totalAddressSpaceSize, alignment, false); if (allocatedMemory.isNull()) { return CEntryPointErrors.RESERVE_ADDRESS_SPACE_FAILED; } @@ -227,7 +226,7 @@ public int initialize(Pointer reservedAddressSpace, UnsignedWord reservedSize, W // Create memory mappings from the image file. UnsignedWord fileOffset = CACHED_IMAGE_HEAP_OFFSET.get().read(); Pointer imageHeap = heapBase.add(imageHeapOffsetInAddressSpace); - imageHeap = VirtualMemoryProvider.get().mapFile(imageHeap, imageHeapSizeInFile, fd, fileOffset, Access.READ, nmtData); + imageHeap = VirtualMemoryProvider.get().mapFile(imageHeap, imageHeapSizeInFile, fd, fileOffset, Access.READ); if (imageHeap.isNull()) { freeImageHeap(allocatedMemory); return CEntryPointErrors.MAP_HEAP_FAILED; @@ -235,16 +234,8 @@ public int initialize(Pointer reservedAddressSpace, UnsignedWord reservedSize, W Pointer relocPointer = IMAGE_HEAP_A_RELOCATABLE_POINTER.get(); ComparableWord relocatedValue = relocPointer.readWord(0); - ComparableWord mappedValue = imageHeap.readWord(relocPointer.subtract(imageHeapBegin)); // *** - // What - // is - // this - // mappedValue? + ComparableWord mappedValue = imageHeap.readWord(relocPointer.subtract(imageHeapBegin)); if (relocatedValue.notEqual(mappedValue)) { - // TODO what is actually happening in this block? - // *** this is checking to see if they've been mapped correctly? [GDB: this doesn't - // actually get entered it seems] - /* * Addresses were relocated by dynamic linker, so copy them, but first remap the pages * to avoid swapping them in from disk. diff --git a/substratevm/src/com.oracle.svm.core.windows/src/com/oracle/svm/core/windows/WindowsImageHeapProvider.java b/substratevm/src/com.oracle.svm.core.windows/src/com/oracle/svm/core/windows/WindowsImageHeapProvider.java index a7195bbe5672..5b06c4420457 100644 --- a/substratevm/src/com.oracle.svm.core.windows/src/com/oracle/svm/core/windows/WindowsImageHeapProvider.java +++ b/substratevm/src/com.oracle.svm.core.windows/src/com/oracle/svm/core/windows/WindowsImageHeapProvider.java @@ -47,7 +47,6 @@ import com.oracle.svm.core.c.function.CEntryPointErrors; import com.oracle.svm.core.headers.LibC; import com.oracle.svm.core.os.AbstractCopyingImageHeapProvider; -import com.oracle.svm.core.nmt.NmtVirtualMemoryData; import com.oracle.svm.core.os.VirtualMemoryProvider; import com.oracle.svm.core.os.VirtualMemoryProvider.Access; import com.oracle.svm.core.windows.headers.FileAPI; @@ -65,18 +64,18 @@ public class WindowsImageHeapProvider extends AbstractCopyingImageHeapProvider { @Override @Uninterruptible(reason = "Called during isolate initialization.") - protected int commitAndCopyMemory(Pointer loadedImageHeap, UnsignedWord imageHeapSize, Pointer newImageHeap, NmtVirtualMemoryData nmtData) { + protected int commitAndCopyMemory(Pointer loadedImageHeap, UnsignedWord imageHeapSize, Pointer newImageHeap) { HANDLE imageHeapFileMapping = getImageHeapFileMapping(); if (imageHeapFileMapping.isNull()) { /* Fall back to copying from memory. */ - return super.commitAndCopyMemory(loadedImageHeap, imageHeapSize, newImageHeap, nmtData); + return super.commitAndCopyMemory(loadedImageHeap, imageHeapSize, newImageHeap); } /* Map a copy-on-write view of the image heap. */ if (VirtualMemoryProvider.get().mapFile(newImageHeap, imageHeapSize, imageHeapFileMapping, getImageHeapFileOffset(), - Access.READ | Access.WRITE, nmtData).isNull()) { + Access.READ | Access.WRITE).isNull()) { /* Fall back to copying from memory. */ - return super.commitAndCopyMemory(loadedImageHeap, imageHeapSize, newImageHeap, nmtData); + return super.commitAndCopyMemory(loadedImageHeap, imageHeapSize, newImageHeap); } /* Copy relocatable pages. */ diff --git a/substratevm/src/com.oracle.svm.core.windows/src/com/oracle/svm/core/windows/WindowsUnmanagedMemorySupportImpl.java b/substratevm/src/com.oracle.svm.core.windows/src/com/oracle/svm/core/windows/WindowsUnmanagedMemorySupportImpl.java index 4e296e5f446b..3f9d85915fea 100644 --- a/substratevm/src/com.oracle.svm.core.windows/src/com/oracle/svm/core/windows/WindowsUnmanagedMemorySupportImpl.java +++ b/substratevm/src/com.oracle.svm.core.windows/src/com/oracle/svm/core/windows/WindowsUnmanagedMemorySupportImpl.java @@ -29,59 +29,52 @@ import org.graalvm.nativeimage.ImageSingletons; import org.graalvm.nativeimage.impl.UnmanagedMemorySupport; import org.graalvm.word.PointerBase; -import org.graalvm.word.Pointer; import org.graalvm.word.UnsignedWord; import org.graalvm.word.WordFactory; import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.feature.AutomaticallyRegisteredImageSingleton; import com.oracle.svm.core.headers.LibCSupport; -import com.oracle.svm.core.nmt.NmtFlag; -import com.oracle.svm.core.nmt.NativeMemoryTracking; @AutomaticallyRegisteredImageSingleton(UnmanagedMemorySupport.class) class WindowsUnmanagedMemorySupportImpl implements UnmanagedMemorySupport { @Override @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public T malloc(UnsignedWord size) { - return malloc(size, NmtFlag.Default.ordinal()); + return libc().malloc(size); } @Override public T malloc(UnsignedWord size, int flag) { - Pointer outerPointer = libc().malloc(size.add(NativeMemoryTracking.getHeaderSize())); - return (T) NativeMemoryTracking.recordMalloc(outerPointer, size, flag); + return libc().malloc(size); } @Override @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public T calloc(UnsignedWord size) { - return calloc(size, NmtFlag.Default.ordinal()); + return libc().calloc(WordFactory.unsigned(1), size); } @Override public T calloc(UnsignedWord size, int flag) { - Pointer outerPointer = libc().calloc(WordFactory.unsigned(1), size.add(NativeMemoryTracking.getHeaderSize())); - return (T) NativeMemoryTracking.recordMalloc(outerPointer, size, flag); + return libc().calloc(WordFactory.unsigned(1), size); } @Override @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public T realloc(T ptr, UnsignedWord size) { - return realloc(ptr, size, NmtFlag.Default.ordinal()); + return libc().realloc(ptr, size); } @Override public T realloc(T ptr, UnsignedWord size, int flag) { - Pointer outerPointer = libc().realloc(ptr, size.add(NativeMemoryTracking.getHeaderSize())); - return (T) NativeMemoryTracking.recordMalloc(outerPointer, size, flag); + return libc().realloc(ptr, size); } @Override @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void free(PointerBase ptr) { - NativeMemoryTracking.deaccountMalloc(ptr); - libc().free(((Pointer) ptr).subtract(NativeMemoryTracking.getHeaderSize())); + libc().free(ptr); } @Override diff --git a/substratevm/src/com.oracle.svm.core.windows/src/com/oracle/svm/core/windows/WindowsVirtualMemoryProvider.java b/substratevm/src/com.oracle.svm.core.windows/src/com/oracle/svm/core/windows/WindowsVirtualMemoryProvider.java index 8ec9f7089b2c..b833a15f32f7 100644 --- a/substratevm/src/com.oracle.svm.core.windows/src/com/oracle/svm/core/windows/WindowsVirtualMemoryProvider.java +++ b/substratevm/src/com.oracle.svm.core.windows/src/com/oracle/svm/core/windows/WindowsVirtualMemoryProvider.java @@ -46,9 +46,6 @@ import com.oracle.svm.core.c.CGlobalDataFactory; import com.oracle.svm.core.c.function.CEntryPointActions; import com.oracle.svm.core.feature.AutomaticallyRegisteredImageSingleton; -import com.oracle.svm.core.nmt.NativeMemoryTracking; -import com.oracle.svm.core.nmt.NmtFlag; -import com.oracle.svm.core.nmt.NmtVirtualMemoryData; import com.oracle.svm.core.os.VirtualMemoryProvider; import com.oracle.svm.core.util.PointerUtils; import com.oracle.svm.core.util.UnsignedUtils; @@ -125,13 +122,10 @@ private static int accessAsProt(int access) { /** Sentinel value indicating that no special alignment is required. */ private static final UnsignedWord UNALIGNED = WordFactory.zero(); - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - public Pointer reserve(UnsignedWord nbytes, UnsignedWord alignment, boolean executable) { - return reserve(nbytes, alignment, executable, WordFactory.nullPointer()); - } + @Override @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - public Pointer reserve(UnsignedWord nbytes, UnsignedWord alignment, boolean executable, NmtVirtualMemoryData nmtData) { + public Pointer reserve(UnsignedWord nbytes, UnsignedWord alignment, boolean executable) { if (nbytes.equal(0)) { return WordFactory.nullPointer(); } @@ -166,11 +160,6 @@ public Pointer reserve(UnsignedWord nbytes, UnsignedWord alignment, boolean exec if (reserved.isNull()) { return WordFactory.nullPointer(); } - if (nmtData.isNull()) { - NativeMemoryTracking.recordReserve(nbytes.add(requiredAlignment), NmtFlag.Default.ordinal()); - } else { - nmtData.setReserved(nmtData.getReserved().add(nbytes.add(requiredAlignment))); - } return requiredAlignment.equal(UNALIGNED) ? reserved : PointerUtils.roundUp(reserved, requiredAlignment); } @@ -295,11 +284,6 @@ private interface MEM_ADDRESS_REQUIREMENTS extends PointerBase { @Override @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public Pointer mapFile(PointerBase start, UnsignedWord nbytes, WordBase fileHandle, UnsignedWord offset, int access) { - return mapFile(start, nbytes, fileHandle, offset, access, WordFactory.nullPointer()); - } - @Override - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - public Pointer mapFile(PointerBase start, UnsignedWord nbytes, WordBase fileHandle, UnsignedWord offset, int access, NmtVirtualMemoryData nmtData) { if ((start.isNonNull() && !isAligned(start)) || nbytes.equal(0)) { return WordFactory.nullPointer(); } @@ -329,11 +313,6 @@ public Pointer mapFile(PointerBase start, UnsignedWord nbytes, WordBase fileHand /* Restore a normal allocation as the caller is unaware of placeholders. */ replacePlaceholder(start, nbytes); } - if (nmtData.isNull()) { - NativeMemoryTracking.recordCommit(nbytes, NmtFlag.Default.ordinal()); - } else { - nmtData.setCommitted(nmtData.getCommitted().add(nbytes)); - } return fileView; } @@ -373,21 +352,10 @@ Pointer invoke(HANDLE fileMapping, HANDLE process, PointerBase baseAddress, long @Override @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public Pointer commit(PointerBase start, UnsignedWord nbytes, int access) { - return commit(start, nbytes, access, WordFactory.nullPointer()); - } - @Override - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - public Pointer commit(PointerBase start, UnsignedWord nbytes, int access, NmtVirtualMemoryData nmtData) { if ((start.isNonNull() && !isAligned(start)) || nbytes.equal(0)) { return WordFactory.nullPointer(); } - if (nmtData.isNull()) { - NativeMemoryTracking.recordCommit(nbytes, NmtFlag.Default.ordinal()); - } else { - nmtData.setCommitted(nmtData.getCommitted().add(nbytes)); - } - /* * VirtualAlloc only guarantees the zeroing for freshly committed pages (i.e., the content * of pages that were already committed earlier won't be touched). diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/Isolates.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/Isolates.java index d8a25b843b8c..1723b3fcba9d 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/Isolates.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/Isolates.java @@ -39,7 +39,6 @@ import com.oracle.svm.core.c.function.CEntryPointSetup; import com.oracle.svm.core.code.CodeInfoTable; import com.oracle.svm.core.heap.Heap; -import com.oracle.svm.core.nmt.NmtVirtualMemoryData; import com.oracle.svm.core.os.CommittedMemoryProvider; import com.oracle.svm.core.util.VMError; @@ -115,9 +114,9 @@ public static int checkIsolate(Isolate isolate) { } @Uninterruptible(reason = "Thread state not yet set up.") - public static int create(WordPointer isolatePointer, CEntryPointCreateIsolateParameters parameters, NmtVirtualMemoryData nmtData) { + public static int create(WordPointer isolatePointer, CEntryPointCreateIsolateParameters parameters) { WordPointer heapBasePointer = StackValue.get(WordPointer.class); - int result = CommittedMemoryProvider.get().initialize(heapBasePointer, parameters, nmtData); + int result = CommittedMemoryProvider.get().initialize(heapBasePointer, parameters); if (result != CEntryPointErrors.NO_ERROR) { return result; } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java index 465ac470c8b1..4298e1455c8d 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java @@ -912,6 +912,9 @@ public Boolean getValue(OptionValues values) { @Option(help = "file:doc-files/FlightRecorderLoggingHelp.txt")// public static final RuntimeOptionKey FlightRecorderLogging = new RuntimeOptionKey<>("all=warning", Immutable); + @Option(help = "Enable native memory tracking")// + public static final RuntimeOptionKey NativeMemoryTracking = new RuntimeOptionKey<>(false); + public static String reportsPath() { Path reportsPath = ImageSingletons.lookup(ReportingSupport.class).reportsPath; if (reportsPath.isAbsolute()) { diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/VMInspectionOptions.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/VMInspectionOptions.java index 90e370c454a6..9e5f61a768b1 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/VMInspectionOptions.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/VMInspectionOptions.java @@ -166,11 +166,7 @@ public static boolean hasThreadDumpSupport() { @Fold public static boolean hasNmtSupport() { - return hasAllOrKeywordMonitoringSupport(MONITORING_NMT_NAME) && !Platform.includedIn(WINDOWS.class); // TODO - // no - // windows - // for - // now? + return hasAllOrKeywordMonitoringSupport(MONITORING_NMT_NAME) && !Platform.includedIn(WINDOWS.class); } @Option(help = "Dumps all runtime compiled methods on SIGUSR2.", type = OptionType.User) // diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/collections/AbstractUninterruptibleHashtable.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/collections/AbstractUninterruptibleHashtable.java index 8eb7cd7b84ac..8166fa91416a 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/collections/AbstractUninterruptibleHashtable.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/collections/AbstractUninterruptibleHashtable.java @@ -42,7 +42,7 @@ public abstract class AbstractUninterruptibleHashtable implements Uninterruptibl private static final int DEFAULT_TABLE_LENGTH = 2053; private final UninterruptibleEntry[] table; - private int size; + protected int size; @Platforms(Platform.HOSTED_ONLY.class) public AbstractUninterruptibleHashtable() { @@ -95,6 +95,26 @@ protected UninterruptibleEntry insertEntry(UninterruptibleEntry valueOnStack) { return WordFactory.nullPointer(); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public void remove(UninterruptibleEntry valueOnStack) { + int index = Integer.remainderUnsigned(valueOnStack.getHash(), DEFAULT_TABLE_LENGTH); + UninterruptibleEntry entry = table[index]; + UninterruptibleEntry prev = WordFactory.nullPointer(); + while (entry.isNonNull()) { + if (isEqual(valueOnStack, entry)) { + if (prev.isNull()) { + table[index] = entry.getNext(); + } else { + prev.setNext(entry.getNext()); + } + free(entry); + return; + } + prev = entry; + entry = entry.getNext(); + } + } + @Override @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public int getSize() { diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/snippets/CEntryPointSnippets.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/snippets/CEntryPointSnippets.java index 7f6f4dbf0cd1..4d11dab84c82 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/snippets/CEntryPointSnippets.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/snippets/CEntryPointSnippets.java @@ -98,10 +98,6 @@ import com.oracle.svm.core.jdk.PlatformNativeLibrarySupport; import com.oracle.svm.core.jdk.RuntimeSupport; import com.oracle.svm.core.log.Log; -import com.oracle.svm.core.nmt.HasNmtSupport; -import com.oracle.svm.core.nmt.NmtFlag; -import com.oracle.svm.core.nmt.NmtVirtualMemoryData; -import com.oracle.svm.core.nmt.NativeMemoryTracking; import com.oracle.svm.core.option.RuntimeOptionParser; import com.oracle.svm.core.os.MemoryProtectionProvider; import com.oracle.svm.core.os.VirtualMemoryProvider; @@ -247,13 +243,7 @@ private static int createIsolate(CEntryPointCreateIsolateParameters providedPara } WordPointer isolatePtr = StackValue.get(WordPointer.class); - NmtVirtualMemoryData nmtData = WordFactory.nullPointer(); - if (HasNmtSupport.get()) { - nmtData = StackValue.get(NmtVirtualMemoryData.class); - nmtData.setCommitted(WordFactory.zero()); - nmtData.setReserved(WordFactory.zero()); - } - int error = Isolates.create(isolatePtr, parameters, nmtData); + int error = Isolates.create(isolatePtr, parameters); if (error != CEntryPointErrors.NO_ERROR) { return error; } @@ -262,11 +252,6 @@ private static int createIsolate(CEntryPointCreateIsolateParameters providedPara setHeapBase(Isolates.getHeapBase(isolate)); } - if (HasNmtSupport.get()) { - NativeMemoryTracking.recordCommit(nmtData.getCommitted(), NmtFlag.Default.ordinal()); - NativeMemoryTracking.recordReserve(nmtData.getReserved(), NmtFlag.Default.ordinal()); - } - return createIsolate0(parsedArgs, isolate, vmThreadSize); } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/SunMiscSubstitutions.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/SunMiscSubstitutions.java index a20bee43c883..ed8e8891c086 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/SunMiscSubstitutions.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/SunMiscSubstitutions.java @@ -46,6 +46,7 @@ import com.oracle.svm.core.hub.DynamicHub; import com.oracle.svm.core.hub.LayoutEncoding; import com.oracle.svm.core.hub.PredefinedClassesSupport; +import com.oracle.svm.core.nmt.NmtFlag; import com.oracle.svm.core.os.VirtualMemoryProvider; import com.oracle.svm.core.util.VMError; @@ -55,12 +56,12 @@ final class Target_jdk_internal_misc_Unsafe_Core { @Substitute private long allocateMemory0(long bytes) { - return UnmanagedMemory.malloc(WordFactory.unsigned(bytes)).rawValue(); + return UnmanagedMemory.malloc(WordFactory.unsigned(bytes), NmtFlag.mtOther.ordinal()).rawValue(); } @Substitute private long reallocateMemory0(long address, long bytes) { - return UnmanagedMemory.realloc(WordFactory.unsigned(address), WordFactory.unsigned(bytes)).rawValue(); + return UnmanagedMemory.realloc(WordFactory.unsigned(address), WordFactory.unsigned(bytes), NmtFlag.mtOther.ordinal()).rawValue(); } @Substitute diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jfr/JfrBufferAccess.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jfr/JfrBufferAccess.java index 31da845e8aa1..f83bd41dd1d0 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jfr/JfrBufferAccess.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jfr/JfrBufferAccess.java @@ -34,6 +34,7 @@ import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.config.ConfigurationValues; +import com.oracle.svm.core.nmt.NmtFlag; import com.oracle.svm.core.thread.VMOperation; import com.oracle.svm.core.util.UnsignedUtils; @@ -61,7 +62,7 @@ public static JfrBuffer allocate(JfrBufferType bufferType) { @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static JfrBuffer allocate(UnsignedWord dataSize, JfrBufferType bufferType) { UnsignedWord headerSize = JfrBufferAccess.getHeaderSize(); - JfrBuffer result = ImageSingletons.lookup(UnmanagedMemorySupport.class).malloc(headerSize.add(dataSize)); + JfrBuffer result = ImageSingletons.lookup(UnmanagedMemorySupport.class).malloc(headerSize.add(dataSize), NmtFlag.mtTracing.ordinal()); if (result.isNonNull()) { result.setSize(dataSize); result.setBufferType(bufferType); diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jfr/JfrBufferNodeAccess.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jfr/JfrBufferNodeAccess.java index 759db4b69ce4..9102ec67abbb 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jfr/JfrBufferNodeAccess.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jfr/JfrBufferNodeAccess.java @@ -34,6 +34,7 @@ import org.graalvm.word.Pointer; import org.graalvm.word.WordFactory; +import com.oracle.svm.core.nmt.NmtFlag; import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.thread.NativeSpinLockUtils; import com.oracle.svm.core.thread.VMOperation; @@ -47,7 +48,7 @@ private JfrBufferNodeAccess() { @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static JfrBufferNode allocate(JfrBuffer buffer) { - JfrBufferNode node = ImageSingletons.lookup(UnmanagedMemorySupport.class).malloc(SizeOf.unsigned(JfrBufferNode.class)); + JfrBufferNode node = ImageSingletons.lookup(UnmanagedMemorySupport.class).malloc(SizeOf.unsigned(JfrBufferNode.class), NmtFlag.mtTracing.ordinal()); if (node.isNonNull()) { node.setBuffer(buffer); node.setNext(WordFactory.nullPointer()); diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jfr/JfrChunkWriter.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jfr/JfrChunkWriter.java index 8e742d63000d..219bdb4c5b7c 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jfr/JfrChunkWriter.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jfr/JfrChunkWriter.java @@ -205,7 +205,6 @@ public void markChunkFinal() { */ public void closeFile() { assert lock.isOwner(); - com.oracle.svm.core.nmt.NativeMemoryTracking.printStats(); /* * Switch to a new epoch. This is done at a safepoint to ensure that we end up with * consistent data, even if multiple threads have JFR events in progress. diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jfr/JfrStackTraceRepository.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jfr/JfrStackTraceRepository.java index 689c3c37f9a0..ee885c2a0c0c 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jfr/JfrStackTraceRepository.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jfr/JfrStackTraceRepository.java @@ -51,6 +51,7 @@ import com.oracle.svm.core.jfr.traceid.JfrTraceIdEpoch; import com.oracle.svm.core.jfr.utils.JfrVisited; import com.oracle.svm.core.locks.VMMutex; +import com.oracle.svm.core.nmt.NmtFlag; import com.oracle.svm.core.sampler.SamplerSampleWriter; import com.oracle.svm.core.sampler.SamplerSampleWriterData; import com.oracle.svm.core.sampler.SamplerSampleWriterDataAccess; @@ -193,7 +194,7 @@ private JfrStackTraceTableEntry getOrPutStackTrace0(Pointer start, UnsignedWord * the thread-local buffer to the C heap because the thread-local buffer will be * overwritten or freed at some point. */ - Pointer to = ImageSingletons.lookup(UnmanagedMemorySupport.class).malloc(size); + Pointer to = ImageSingletons.lookup(UnmanagedMemorySupport.class).malloc(size, NmtFlag.mtTracing.ordinal()); if (to.isNonNull()) { UnmanagedMemoryUtil.copy(start, to, size); entry.setRawStackTrace(to); diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/MallocHeader.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/MallocHeader.java index 84c680b35dd8..f0b6929fd15b 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/MallocHeader.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/MallocHeader.java @@ -30,6 +30,11 @@ import org.graalvm.nativeimage.c.struct.RawStructure; import org.graalvm.word.PointerBase; +/** + * A "malloc header" is used to cache metadata about the native allocation (calloc/realloc/malloc). + * To do this, a small amount of additional space is requested contiguous to the user allocation. + * This metadata is later used to update the memory tracking once the block is freed. + */ @RawStructure public interface MallocHeader extends PointerBase { @RawField diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/MallocMemoryInfo.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/MallocMemoryInfo.java index 4393b5b93b2f..c2159078129a 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/MallocMemoryInfo.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/MallocMemoryInfo.java @@ -51,8 +51,9 @@ void recordMalloc(UnsignedWord allocationSize) { @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) void deaccountMalloc(long allocationSize) { - count.decrementAndGet(); - size.addAndGet(-allocationSize); + long lastCount = count.decrementAndGet(); + long lastSize = size.addAndGet(-allocationSize); + assert lastSize >= 0 && lastCount >= 0; } long getSize() { diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/NativeMemoryTracking.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/NativeMemoryTracking.java index 807185a0725b..984731c961be 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/NativeMemoryTracking.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/NativeMemoryTracking.java @@ -26,67 +26,167 @@ package com.oracle.svm.core.nmt; +import com.oracle.svm.core.SubstrateOptions; import com.oracle.svm.core.Uninterruptible; +import com.oracle.svm.core.config.ConfigurationValues; +import com.oracle.svm.core.headers.LibCSupport; +import com.oracle.svm.core.jdk.RuntimeSupport; +import com.oracle.svm.core.thread.JavaSpinLockUtils; import com.oracle.svm.core.util.UnsignedUtils; +import com.oracle.svm.util.LogUtils; + +import jdk.graal.compiler.api.replacements.Fold; +import jdk.internal.misc.Unsafe; + +import org.graalvm.nativeimage.ImageSingletons; +import org.graalvm.nativeimage.Platform; +import org.graalvm.nativeimage.Platforms; import org.graalvm.nativeimage.c.struct.SizeOf; -import org.graalvm.word.WordFactory; -import com.oracle.svm.core.config.ConfigurationValues; +import org.graalvm.nativeimage.impl.UnmanagedMemorySupport; import org.graalvm.word.Pointer; import org.graalvm.word.PointerBase; import org.graalvm.word.UnsignedWord; +import org.graalvm.word.WordFactory; -import org.graalvm.nativeimage.Platforms; -import org.graalvm.nativeimage.Platform; +/** + * This is the class that handles native memory tracking (NMT). There are two components to NMT: + * tracking malloc/realloc/calloc, and tracking virtual memory usage. + *

+ *

+ *

+ * Malloc/realloc/calloc: Malloc headers ({@link MallocHeader}) are used for caching data about the + * allocation. This is necessary because that data is needed to deaccount the tracked memory when it + * is freed. There are some scenarios where an implementation dependent on malloc header requires + * special precautions: + *

+ * + *
    + *
  1. When native memory is allocated outside of java. This might be in a C library for example. + * When it comes time to free the memory, it is important that the NMT infrastructure, if enabled, + * does not assume a malloc header exists and attempt to access it. + * {@link UnmanagedMemorySupport#untrackedFree(PointerBase)} must be used in this case. + *
  2. When raw {@link LibCSupport} allocation methods are called directly. This is problematic + * because such allocated blocks will not contain malloc headers. If NMT is enabled it will assume + * headers exist and access incorrect memory locations. This must be avoided. + *
  3. During the phase pre-NMT initialization. Before runtime options are parsed, it is unknown + * whether NMT should be enabled/disabled. During this time, it is assumed that NMT will later be + * enabled, so malloc headers are allocated. This increases the accuracy of the tracking. However, + * if NMT is later disabled upon initialization, there will be a mix of allocated blocks that + * have headers and do some that do not . This is a problem when it comes time to free the + * blocks. For this reason, there are methods in this class that handle pre-init allocations using a + * look-up table to cache addresses seen before initialization. This is the same reason NMT cannot + * be enabled/disabled after initialization. + *
  4. If native allocations are done before the image heap is mapped. This is not strictly a + * problem related to malloc headers, but must still be avoided. If such situations are unavoidable, + * NMT code must be bypassed using raw {@link LibCSupport} methods for both allocation and + * deallocation. + *
+ */ +public class NativeMemoryTracking { + private static final Unsafe U = Unsafe.getUnsafe(); + private static final long LOCK_OFFSET = U.objectFieldOffset(NativeMemoryTracking.class, "preInitLock"); -import org.graalvm.nativeimage.ImageSingletons; + /** + * Can't use VmMutex because it tracks owners and this class may be used by unattached threads + * (calloc). This lock is only used during the pre-init phase. + */ + @SuppressWarnings("unused") private volatile int preInitLock; -public class NativeMemoryTracking { - public com.oracle.svm.core.nmt.MallocMemorySnapshot mallocMemorySnapshot; - private VirtualMemorySnapshot virtualMemorySnapshot; - private boolean enabled; + private final PreInitTable preInitTable; + private volatile boolean enabled; + private volatile boolean initialized; + + private MallocMemorySnapshot mallocMemorySnapshot; @Platforms(Platform.HOSTED_ONLY.class) public NativeMemoryTracking() { - enabled = true; // TODO enable via the same runtime options in hotspot + enabled = true; + initialized = false; mallocMemorySnapshot = new MallocMemorySnapshot(); - virtualMemorySnapshot = new VirtualMemorySnapshot(); + preInitTable = new PreInitTable(); + } + + @Fold + static UnsignedWord getHeaderSize0() { + return UnsignedUtils.roundUp(SizeOf.unsigned(MallocHeader.class), WordFactory.unsigned(ConfigurationValues.getTarget().wordSize)); + } + + @Fold + static LibCSupport libc() { + return ImageSingletons.lookup(LibCSupport.class); + } + + @Fold + static NativeMemoryTracking get() { + return ImageSingletons.lookup(NativeMemoryTracking.class); + } + + /** This must be called after the image heap is mapped. */ + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public static void initialize(boolean enabled) { + if (HasNmtSupport.get()) { + get().initialize0(enabled); + } } @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - public static void setEnabled(boolean enabled) { - get().enabled = enabled; + private void initialize0(boolean enabled) { + assert !initialized; + // Block until all in-progress pre-init operations are completed + lockNoTransition(); + try { + this.enabled = enabled; + // After this point, the preInit table becomes read only. + initialized = true; + } finally { + unlock(); + } } @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static UnsignedWord getHeaderSize() { - if (HasNmtSupport.get() && get().enabled) { // *** SubstrateJVM calls HasJfrSupport - return UnsignedUtils.roundUp(SizeOf.unsigned(MallocHeader.class), WordFactory.unsigned(ConfigurationValues.getTarget().wordSize)); + if (HasNmtSupport.get() && get().enabled) { + return getHeaderSize0(); } return WordFactory.zero(); } @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - private static NativeMemoryTracking get() { - return ImageSingletons.lookup(NativeMemoryTracking.class); + public static long getAllocationSize(Pointer outerPointer) { + if (HasNmtSupport.get() && get().enabled) { + MallocHeader header = (MallocHeader) outerPointer; + return header.getSize(); + } + return -1; } @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - public static Pointer recordMalloc(Pointer outerPointer, UnsignedWord size, int flag) { + public static int getAllocationCategory(Pointer outerPointer) { if (HasNmtSupport.get() && get().enabled) { - get().mallocMemorySnapshot.getInfoByCategory(flag).recordMalloc(size); - get().mallocMemorySnapshot.getTotalInfo().recordMalloc(size); - return initializeHeader(outerPointer, size, flag); + MallocHeader header = (MallocHeader) outerPointer; + return header.getFlag(); + } + return -1; + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public static Pointer recordMalloc(Pointer outerPointer, UnsignedWord size, int flag) { + if (HasNmtSupport.get() && outerPointer.isNonNull()) { + return get().recordMalloc0(outerPointer, size, flag); } return outerPointer; } @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - public static void deaccountMalloc(PointerBase ptr) { - if (HasNmtSupport.get() && get().enabled) { - MallocHeader header = (MallocHeader) ((Pointer) ptr).subtract(getHeaderSize()); - get().mallocMemorySnapshot.getInfoByCategory(header.getFlag()).deaccountMalloc(header.getSize()); - get().mallocMemorySnapshot.getTotalInfo().deaccountMalloc(header.getSize()); + private Pointer recordMalloc0(Pointer outerPointer, UnsignedWord size, int flag) { + if (enabled) { + UnsignedWord headerSize = getHeaderSize0(); + mallocMemorySnapshot.getInfoByCategory(flag).recordMalloc(size); + mallocMemorySnapshot.getInfoByCategory(NmtFlag.mtNMT.ordinal()).recordMalloc(headerSize); + mallocMemorySnapshot.getTotalInfo().recordMalloc(size.add(headerSize)); + return initializeHeader(outerPointer, size, flag); } + return outerPointer; } @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) @@ -94,52 +194,304 @@ private static Pointer initializeHeader(Pointer outerPointer, UnsignedWord size, MallocHeader mallocHeader = (MallocHeader) outerPointer; mallocHeader.setSize(size.rawValue()); mallocHeader.setFlag(flag); - return ((Pointer) mallocHeader).add(NativeMemoryTracking.getHeaderSize()); + return ((Pointer) mallocHeader).add(NativeMemoryTracking.getHeaderSize0()); } + /** + * This is only needed for {@link PreInitTable} since we must use raw malloc/free there but + * still want to track its allocations. + */ @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - public static void recordReserve(UnsignedWord size, int flag) { - if (HasNmtSupport.get() && get().enabled) { - get().virtualMemorySnapshot.getInfoByCategory(flag).recordReserved(size); - get().virtualMemorySnapshot.getTotalInfo().recordReserved(size); + public static void recordMallocWithoutHeader(UnsignedWord size, int flag) { + if (HasNmtSupport.get()) { + get().recordMallocWithoutHeader0(size, flag); } } @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - public static void recordCommit(UnsignedWord size, int flag) { - if (HasNmtSupport.get() && get().enabled) { - get().virtualMemorySnapshot.getInfoByCategory(flag).recordCommitted(size); - get().virtualMemorySnapshot.getTotalInfo().recordCommitted(size); + private void recordMallocWithoutHeader0(UnsignedWord size, int flag) { + if (enabled) { + mallocMemorySnapshot.getInfoByCategory(flag).recordMalloc(size); + mallocMemorySnapshot.getTotalInfo().recordMalloc(size); } } @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - public static void recordUncommit(UnsignedWord size, int flag) { + public static void deaccountMalloc(PointerBase innerPtr) { if (HasNmtSupport.get() && get().enabled) { - get().virtualMemorySnapshot.getInfoByCategory(flag).recordUncommit(size); - get().virtualMemorySnapshot.getTotalInfo().recordUncommit(size); + MallocHeader header = (MallocHeader) ((Pointer) innerPtr).subtract(getHeaderSize0()); + get().deaccountMalloc0(header.getSize(), header.getFlag()); } } @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - public static void recordFree(UnsignedWord size, int flag) { - if (HasNmtSupport.get() && get().enabled) { - get().virtualMemorySnapshot.getInfoByCategory(flag).recordFree(size); - get().virtualMemorySnapshot.getTotalInfo().recordFree(size); + public static void deaccountMalloc(long size, int flag) { + if (HasNmtSupport.get() && size > 0 && flag >= 0) { + get().deaccountMalloc0(size, flag); + } + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + private void deaccountMalloc0(long size, int flag) { + if (enabled) { + mallocMemorySnapshot.getInfoByCategory(flag).deaccountMalloc(size); + mallocMemorySnapshot.getInfoByCategory(NmtFlag.mtNMT.ordinal()).deaccountMalloc(getHeaderSize0().rawValue()); + mallocMemorySnapshot.getTotalInfo().deaccountMalloc(size + getHeaderSize0().rawValue()); + } + } + + /** + * This is only needed for {@link PreInitTable} since we must use raw malloc/free there but + * still want to track its allocations. + */ + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public static void deaccountMallocWithoutHeader(long size, int flag) { + if (HasNmtSupport.get() && size > 0 && flag >= 0) { + get().deaccountMallocWithoutHeader0(size, flag); + } + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + private void deaccountMallocWithoutHeader0(long size, int flag) { + if (enabled) { + mallocMemorySnapshot.getInfoByCategory(flag).deaccountMalloc(size); + mallocMemorySnapshot.getTotalInfo().deaccountMalloc(size); + } + } + + public static long getMallocByCategory(NmtFlag flag) { + if (HasNmtSupport.get()) { + return get().mallocMemorySnapshot.getInfoByCategory(flag.ordinal()).getSize(); } + return -1; } + /** Prints stats contained in the current snapshot. */ public static void printStats() { if (HasNmtSupport.get()) { - System.out.println("MALLOC"); - for (int i = 0; i < com.oracle.svm.core.nmt.NmtFlag.values().length; i++) { + get().printStats0(); + } + } + + private void printStats0() { + if (!enabled) { + return; + } + LogUtils.info("Total current malloc size:" + mallocMemorySnapshot.getTotalInfo().getSize() + "B"); + LogUtils.info("Total current malloc count:" + mallocMemorySnapshot.getTotalInfo().getCount()); + + for (int i = 0; i < NmtFlag.values().length; i++) { + LogUtils.info(NmtFlag.values()[i].getName() + " current malloc size:" + mallocMemorySnapshot.getInfoByCategory(i).getSize() + "B"); + LogUtils.info(NmtFlag.values()[i].getName() + " current malloc count:" + mallocMemorySnapshot.getInfoByCategory(i).getCount()); + } + + } + + /** + * This method is needed because locking is required to make the allocation, LUT insertion, and + * initialization check atomic. If the malloc was handled here, returnAddress holds the location + * of the malloc block payload. + */ + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public static boolean handlePreInitMallocs(UnsignedWord size, int flag, ReturnAddress returnAddress) { + if (HasNmtSupport.get()) { + return get().handlePreInitMallocs0(size, flag, returnAddress); + } + return false; + } + /** + * Special handling is needed here becuase we need ot check initialization state and add an + * address to the LUT. + */ + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + private boolean handlePreInitMallocs0(UnsignedWord size, int flag, ReturnAddress returnAddress) { + // For speed, check if initialized before acquiring lock + if (!initialized) { + lockNoTransition(); + try { + // Double check after acquiring lock. If it's since been initialized, proceed + // normally. + if (initialized) { + return false; + } + // Still uninitialized. Allocate and add to LUT. + Pointer outerPointer = libc().malloc(size.add(getHeaderSize0())); + recordPreInitAlloc(outerPointer, size, flag, returnAddress); + return true; + } finally { + unlock(); } - System.out.println("total size:" + get().mallocMemorySnapshot.getTotalInfo().getSize()); - System.out.println("total count:" + get().mallocMemorySnapshot.getTotalInfo().getCount()); - System.out.println("Virtual"); - System.out.println("total reserved:" + get().virtualMemorySnapshot.getTotalInfo().getReservedSize()); - System.out.println("total committed:" + get().virtualMemorySnapshot.getTotalInfo().getCommittedSize()); } + return false; + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public static boolean handlePreInitCallocs(UnsignedWord size, int flag, ReturnAddress returnAddress) { + if (HasNmtSupport.get()) { + return get().handlePreInitCallocs0(size, flag, returnAddress); + } + return false; + } + + /** + * Similar to + * {@link NativeMemoryTracking#handlePreInitMallocs0(UnsignedWord, int, ReturnAddress)}. + */ + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + private boolean handlePreInitCallocs0(UnsignedWord size, int flag, ReturnAddress returnAddress) { + if (!initialized) { + lockNoTransition(); + try { + if (initialized) { + return false; + } + Pointer outerPointer = libc().calloc(WordFactory.unsigned(1), size.add(getHeaderSize())); + recordPreInitAlloc(outerPointer, size, flag, returnAddress); + return true; + } finally { + unlock(); + } + } + return false; + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public static boolean handlePreInitReallocs(Pointer oldInnerPtr, UnsignedWord size, int flag, ReturnAddress returnAddress) { + if (HasNmtSupport.get()) { + return get().handlePreInitReallocs0(oldInnerPtr, size, flag, returnAddress); + } + return false; + } + + /** + * This method is needed because locking is required to make the allocation, LUT insertion, and + * initialization check atomic. + */ + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + private boolean handlePreInitReallocs0(Pointer oldInnerPtr, UnsignedWord size, int flag, ReturnAddress returnAddress) { + // For speed, check if initialized before acquiring lock + if (!initialized) { + lockNoTransition(); + try { + // Double check after acquiring lock. If still uninitialized, perform realloc and do + // LUT updates. + if (!initialized) { + // Retrieve necessary data from the old block + Pointer oldOuterPointer = oldInnerPtr.subtract(NativeMemoryTracking.getHeaderSize0()); + long oldSize = getAllocationSize(oldOuterPointer); + int oldCategory = getAllocationCategory(oldOuterPointer); + + // Perform the realloc + Pointer newOuterPointer = libc().realloc(oldOuterPointer, size.add(NativeMemoryTracking.getHeaderSize0())); + + if (newOuterPointer.isNonNull()) { + // Deaccount old block. Order matters here in case old/new have same + // address. + deaccountMalloc0(oldSize, oldCategory); + preInitTable.remove(oldInnerPtr); + + // Account new block + recordPreInitAlloc(newOuterPointer, size, flag, returnAddress); + } else { + returnAddress.set(WordFactory.nullPointer()); + } + return true; + } + } finally { + unlock(); + } + } + + // Post init. + if (!enabled && preInitTable.get(oldInnerPtr).isNonNull()) { + // There is a header from pre-init time, but tracking has since been disabled. + + // Malloc a new block with the given size and no header + Pointer newBlock = libc().malloc(size); + returnAddress.set(newBlock); + if (newBlock.isNonNull()) { + // Copy payload from original block to new block + Pointer oldOuterPointer = oldInnerPtr.subtract(getHeaderSize0()); + UnsignedWord oldSize = WordFactory.unsigned(getAllocationSize(oldOuterPointer)); + UnsignedWord amountToCopy = size.belowThan(oldSize) ? size : oldSize; + libc().memcpy(newBlock, oldInnerPtr, amountToCopy); + // Don't raw free the old block to avoid libc returning the same address later. + } + return true; + } + return false; + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + private void recordPreInitAlloc(Pointer newOuterPointer, UnsignedWord size, int flag, ReturnAddress returnAddress) { + if (newOuterPointer.isNull()) { + returnAddress.set(WordFactory.nullPointer()); + return; + } + Pointer newInnerPtr = recordMalloc0(newOuterPointer, size, flag); + preInitTable.putIfAbsent(newInnerPtr); + returnAddress.set(newInnerPtr); + } + + /** Special handling is needed for pre-init mallocs if NMT is disabled post-init. */ + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public static boolean handlePreInitFrees(Pointer innerPtr) { + if (HasNmtSupport.get()) { + return get().handlePreInitFrees0(innerPtr); + } + return false; + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + private boolean handlePreInitFrees0(Pointer innerPtr) { + if (!initialized) { + lockNoTransition(); + try { + if (!initialized) { + deaccountMalloc(innerPtr); + preInitTable.remove(innerPtr); + libc().free(innerPtr.subtract(getHeaderSize0())); + return true; + } + } finally { + unlock(); + } + } + + // Post init. + if (!enabled && preInitTable.get(innerPtr).isNonNull()) { + // If NMT is now disabled and we're dealing with a pre-init block. + // There is a header from pre-init time, but tracking has since been disabled. + // Do nothing more. Don't Raw free the old block to avoid libc returning the same + // address later. + return true; + + } + // Either NMT is still enabled or we're dealing with a block allocated after initialization. + // We can proceed with tracking normally. + return false; + } + + @Uninterruptible(reason = "Locking without transition requires that the whole critical section is uninterruptible.", callerMustBe = true) + private void lockNoTransition() { + JavaSpinLockUtils.lockNoTransition(this, LOCK_OFFSET); + } + + @Uninterruptible(reason = "Locking without transition requires that the whole critical section is uninterruptible.", callerMustBe = true) + private void unlock() { + JavaSpinLockUtils.unlock(this, LOCK_OFFSET); + } + + public static RuntimeSupport.Hook shutdownHook() { + return isFirstIsolate -> { + printStats(); + }; + } + + public static RuntimeSupport.Hook startupHook() { + return isFirstIsolate -> { + initialize(SubstrateOptions.NativeMemoryTracking.getValue()); + }; } } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/NmtFeature.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/NmtFeature.java index 6b442f9be95d..ca6d96b244a6 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/NmtFeature.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/NmtFeature.java @@ -31,16 +31,30 @@ import com.oracle.svm.core.feature.AutomaticallyRegisteredFeature; import com.oracle.svm.core.feature.InternalFeature; - +import com.oracle.svm.core.jdk.RuntimeSupport; @AutomaticallyRegisteredFeature public class NmtFeature implements InternalFeature { @Override public boolean isInConfiguration(IsInConfigurationAccess access) { + return isInConfiguration(); + } + + public static boolean isInConfiguration() { return VMInspectionOptions.hasNmtSupport(); } + @Override public void afterRegistration(AfterRegistrationAccess access) { ImageSingletons.add(NativeMemoryTracking.class, new NativeMemoryTracking()); } + + @Override + public void beforeAnalysis(BeforeAnalysisAccess access) { + RuntimeSupport runtime = RuntimeSupport.getRuntimeSupport(); + + runtime.addShutdownHook(NativeMemoryTracking.shutdownHook()); + runtime.addStartupHook(NativeMemoryTracking.startupHook()); + } + } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/NmtFlag.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/NmtFlag.java index 0b6059934081..7f3a371117ca 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/NmtFlag.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/NmtFlag.java @@ -27,10 +27,20 @@ import com.oracle.svm.core.Uninterruptible; +/** These category flag names match their counterparts in Hotspot. */ public enum NmtFlag { - Jfr("jfr"), - Nmt("native-memory-tracking"), - Default("untracked"); + mtJavaHeap("Java Heap"), + mtThread("Thread"), + mtThreadStack("Thread Stack"), + mtServiceability("Serviceability"), + mtGC("GC"), + mtInternal("Internal"), // Memory used by VM, outside other categories + mtCode("Code"), + mtOther("Other"), // Memory not used by VM (Unsafe) + mtNMT("Native Memory Tracking"), // Memory used by NMT itself + mtTest("Test"), // Test type for verifying NMT + mtTracing("Tracing"), // JFR + mtNone("Unknown"); // This is the default category private final String name; diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/VirtualMemorySnapshot.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/NmtTestFeature.java similarity index 55% rename from substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/VirtualMemorySnapshot.java rename to substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/NmtTestFeature.java index 59e8b17a0ad8..cedbdf21a0b3 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/VirtualMemorySnapshot.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/NmtTestFeature.java @@ -26,33 +26,18 @@ package com.oracle.svm.core.nmt; -import com.oracle.svm.core.Uninterruptible; -import org.graalvm.nativeimage.Platform; -import org.graalvm.nativeimage.Platforms; +import org.graalvm.nativeimage.hosted.Feature; +import org.graalvm.nativeimage.ImageSingletons; -class VirtualMemorySnapshot { // TODO this is the same as MallocMemorySnapshot. Maybe we can reduce - // code duplication. Lets do that last once we know ALL the methods - // these classes we'll need - private VirtualMemoryInfo[] categories; - private VirtualMemoryInfo total; - - @Platforms(Platform.HOSTED_ONLY.class) - VirtualMemorySnapshot() { - total = new VirtualMemoryInfo(); - categories = new VirtualMemoryInfo[NmtFlag.values().length]; - for (int i = 0; i < categories.length; i++) { - categories[i] = new VirtualMemoryInfo(); - } - } - - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - VirtualMemoryInfo getInfoByCategory(int flag) { - assert flag < categories.length; - return categories[flag]; +/** This feature is required to omit the startup and shutdown hooks. */ +class NmtTestFeature implements Feature { + @Override + public void afterRegistration(AfterRegistrationAccess access) { + ImageSingletons.add(NativeMemoryTracking.class, new NativeMemoryTracking()); } - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - VirtualMemoryInfo getTotalInfo() { - return total; + @Override + public boolean isInConfiguration(IsInConfigurationAccess access) { + return !NmtFeature.isInConfiguration(); } } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/PreInitTable.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/PreInitTable.java new file mode 100644 index 000000000000..e5d61f036e0b --- /dev/null +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/PreInitTable.java @@ -0,0 +1,102 @@ +/* + * Copyright (c) 2023, 2023, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2023, 2023, Red Hat Inc. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +package com.oracle.svm.core.nmt; + +import com.oracle.svm.core.Uninterruptible; +import com.oracle.svm.core.UnmanagedMemoryUtil; +import com.oracle.svm.core.collections.UninterruptibleEntry; +import com.oracle.svm.core.collections.AbstractUninterruptibleHashtable; +import com.oracle.svm.core.headers.LibCSupport; +import org.graalvm.nativeimage.c.struct.SizeOf; +import org.graalvm.word.Pointer; +import org.graalvm.word.UnsignedWord; +import org.graalvm.word.WordFactory; + +import org.graalvm.nativeimage.ImageSingletons; +import org.graalvm.nativeimage.StackValue; +import com.oracle.svm.core.jdk.UninterruptibleUtils; + +/** + * This table stores addresses of NMT pre-init malloc blocks. These blocks will always contain + * headers. It is important that this class only uses raw malloc/free to avoid recursive behaviour. + * This table is read-only after NMT is initialized to mitigate the cost of synchronization. + */ +class PreInitTable extends AbstractUninterruptibleHashtable { + + @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + protected UninterruptibleEntry[] createTable(int length) { + return new UninterruptibleEntry[length]; + } + + @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + protected boolean isEqual(UninterruptibleEntry a, UninterruptibleEntry b) { + return a.getHash() == b.getHash(); + } + + @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + protected UninterruptibleEntry copyToHeap(UninterruptibleEntry valueOnStack) { + UnsignedWord size = SizeOf.unsigned(UninterruptibleEntry.class); + UninterruptibleEntry pointerOnHeap = ImageSingletons.lookup(LibCSupport.class).malloc(size); + if (pointerOnHeap.isNonNull()) { + NativeMemoryTracking.recordMallocWithoutHeader(size, NmtFlag.mtNMT.ordinal()); + UnmanagedMemoryUtil.copy((Pointer) valueOnStack, (Pointer) pointerOnHeap, SizeOf.unsigned(UninterruptibleEntry.class)); + return pointerOnHeap; + } + return WordFactory.nullPointer(); + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + protected void free(UninterruptibleEntry entry) { + size--; + ImageSingletons.lookup(LibCSupport.class).free(entry); + NativeMemoryTracking.deaccountMallocWithoutHeader(SizeOf.unsigned(UninterruptibleEntry.class).rawValue(), NmtFlag.mtNMT.ordinal()); + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public void remove(Pointer ptr) { + UninterruptibleEntry entry = StackValue.get(UninterruptibleEntry.class); + entry.setHash(UninterruptibleUtils.Long.hashCode(ptr.rawValue())); + remove(entry); + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public boolean putIfAbsent(Pointer ptr) { + UninterruptibleEntry entry = StackValue.get(UninterruptibleEntry.class); + entry.setHash(UninterruptibleUtils.Long.hashCode(ptr.rawValue())); + return putIfAbsent(entry); + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public UninterruptibleEntry get(Pointer ptr) { + UninterruptibleEntry entry = StackValue.get(UninterruptibleEntry.class); + entry.setHash(UninterruptibleUtils.Long.hashCode(ptr.rawValue())); + return get(entry); + } +} diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/NmtVirtualMemoryData.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/ReturnAddress.java similarity index 84% rename from substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/NmtVirtualMemoryData.java rename to substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/ReturnAddress.java index 2d98a1c80725..e8820ecc8e57 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/NmtVirtualMemoryData.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/ReturnAddress.java @@ -28,20 +28,15 @@ import org.graalvm.nativeimage.c.struct.RawField; import org.graalvm.nativeimage.c.struct.RawStructure; + +import org.graalvm.word.Pointer; import org.graalvm.word.PointerBase; -import org.graalvm.word.UnsignedWord; @RawStructure -public interface NmtVirtualMemoryData extends PointerBase { - @RawField - UnsignedWord getReserved(); - - @RawField - void setReserved(UnsignedWord value); - +public interface ReturnAddress extends PointerBase { @RawField - UnsignedWord getCommitted(); + Pointer get(); @RawField - void setCommitted(UnsignedWord value); + void set(Pointer value); } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/VirtualMemoryInfo.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/VirtualMemoryInfo.java deleted file mode 100644 index 9ac7e07d5e30..000000000000 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/VirtualMemoryInfo.java +++ /dev/null @@ -1,74 +0,0 @@ -/* - * Copyright (c) 2023, 2023, Oracle and/or its affiliates. All rights reserved. - * Copyright (c) 2023, 2023, Red Hat Inc. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. Oracle designates this - * particular file as subject to the "Classpath" exception as provided - * by Oracle in the LICENSE file that accompanied this code. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - */ - -package com.oracle.svm.core.nmt; - -import com.oracle.svm.core.Uninterruptible; -import com.oracle.svm.core.jdk.UninterruptibleUtils.AtomicLong; -import org.graalvm.nativeimage.Platform; -import org.graalvm.nativeimage.Platforms; -import org.graalvm.word.UnsignedWord; - -class VirtualMemoryInfo { // TODO track peak - private AtomicLong reservedSize; - private AtomicLong committedSize; - - @Platforms(Platform.HOSTED_ONLY.class) - VirtualMemoryInfo() { - reservedSize = new AtomicLong(0); - committedSize = new AtomicLong(0); - } - - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - void recordReserved(UnsignedWord size) { - reservedSize.addAndGet(size.rawValue()); - } - - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - void recordCommitted(UnsignedWord size) { // *** hotspot doesnt adjust reserved when mem is - // committed. The same block is counted as both - // reserved and committed. - committedSize.addAndGet(size.rawValue()); - } - - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - void recordUncommit(UnsignedWord size) { - committedSize.addAndGet(-size.rawValue()); - } - - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - void recordFree(UnsignedWord size) { - reservedSize.addAndGet(-size.rawValue()); - } - - long getReservedSize() { - return reservedSize.get(); - } - - long getCommittedSize() { - return committedSize.get(); - } -} diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/os/AbstractCopyingImageHeapProvider.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/os/AbstractCopyingImageHeapProvider.java index c1c5399dc95e..afe8fde670ff 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/os/AbstractCopyingImageHeapProvider.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/os/AbstractCopyingImageHeapProvider.java @@ -40,7 +40,6 @@ import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.c.function.CEntryPointErrors; import com.oracle.svm.core.heap.Heap; -import com.oracle.svm.core.nmt.NmtVirtualMemoryData; import com.oracle.svm.core.os.VirtualMemoryProvider.Access; import com.oracle.svm.core.util.UnsignedUtils; @@ -52,7 +51,7 @@ public boolean guaranteesHeapPreferredAddressSpaceAlignment() { @Override @Uninterruptible(reason = "Called during isolate initialization.") - public int initialize(Pointer reservedAddressSpace, UnsignedWord reservedSize, WordPointer basePointer, WordPointer endPointer, NmtVirtualMemoryData nmtData) { + public int initialize(Pointer reservedAddressSpace, UnsignedWord reservedSize, WordPointer basePointer, WordPointer endPointer) { boolean haveDynamicMethodResolution = DynamicMethodAddressResolutionHeapSupport.isEnabled(); UnsignedWord preHeapRequiredBytes = WordFactory.zero(); UnsignedWord alignment = WordFactory.unsigned(Heap.getHeap().getPreferredAddressSpaceAlignment()); @@ -72,7 +71,7 @@ public int initialize(Pointer reservedAddressSpace, UnsignedWord reservedSize, W Pointer heapBase; Pointer allocatedMemory = WordFactory.nullPointer(); if (reservedAddressSpace.isNull()) { - heapBase = allocatedMemory = VirtualMemoryProvider.get().reserve(totalAddressSpaceSize, alignment, false, nmtData); + heapBase = allocatedMemory = VirtualMemoryProvider.get().reserve(totalAddressSpaceSize, alignment, false); if (allocatedMemory.isNull()) { return CEntryPointErrors.RESERVE_ADDRESS_SPACE_FAILED; } @@ -101,7 +100,7 @@ public int initialize(Pointer reservedAddressSpace, UnsignedWord reservedSize, W Word imageHeapBegin = IMAGE_HEAP_BEGIN.get(); UnsignedWord imageHeapSizeInFile = getImageHeapSizeInFile(); Pointer imageHeap = heapBase.add(Heap.getHeap().getImageHeapOffsetInAddressSpace()); - int result = commitAndCopyMemory(imageHeapBegin, imageHeapSizeInFile, imageHeap, nmtData); + int result = commitAndCopyMemory(imageHeapBegin, imageHeapSizeInFile, imageHeap); if (result != CEntryPointErrors.NO_ERROR) { freeImageHeap(allocatedMemory); return result; @@ -146,8 +145,8 @@ public int initialize(Pointer reservedAddressSpace, UnsignedWord reservedSize, W } @Uninterruptible(reason = "Called during isolate initialization.") - protected int commitAndCopyMemory(Pointer loadedImageHeap, UnsignedWord imageHeapSize, Pointer newImageHeap, NmtVirtualMemoryData nmtData) { - Pointer actualNewImageHeap = VirtualMemoryProvider.get().commit(newImageHeap, imageHeapSize, Access.READ | Access.WRITE, nmtData); + protected int commitAndCopyMemory(Pointer loadedImageHeap, UnsignedWord imageHeapSize, Pointer newImageHeap) { + Pointer actualNewImageHeap = VirtualMemoryProvider.get().commit(newImageHeap, imageHeapSize, Access.READ | Access.WRITE); if (actualNewImageHeap.isNull() || actualNewImageHeap.notEqual(newImageHeap)) { return CEntryPointErrors.RESERVE_ADDRESS_SPACE_FAILED; } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/os/CommittedMemoryProvider.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/os/CommittedMemoryProvider.java index 802de4a1841e..c1111a02654c 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/os/CommittedMemoryProvider.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/os/CommittedMemoryProvider.java @@ -36,7 +36,6 @@ import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.c.function.CEntryPointCreateIsolateParameters; import com.oracle.svm.core.heap.Heap; -import com.oracle.svm.core.nmt.NmtVirtualMemoryData; /** * A provider of ranges of committed memory, which is virtual memory that is backed by physical @@ -63,7 +62,7 @@ static CommittedMemoryProvider get() { * @return zero in case of success, non-zero in case of an error. */ @Uninterruptible(reason = "Still being initialized.") - int initialize(WordPointer heapBasePointer, CEntryPointCreateIsolateParameters parameters, NmtVirtualMemoryData nmtData); + int initialize(WordPointer heapBasePointer, CEntryPointCreateIsolateParameters parameters); /** * Tear down for the current isolate. This must be the last method of this interface diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/os/ImageHeapProvider.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/os/ImageHeapProvider.java index 2a4dc8b78e6a..b277db95d154 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/os/ImageHeapProvider.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/os/ImageHeapProvider.java @@ -33,7 +33,6 @@ import com.oracle.svm.core.c.function.CEntryPointErrors; import com.oracle.svm.core.heap.Heap; -import com.oracle.svm.core.nmt.NmtVirtualMemoryData; /** * Provides new instances of the image heap for creating isolates. The same image heap provider @@ -80,7 +79,7 @@ static ImageHeapProvider get() { * written. May be null if this value is not required. * @return a result code from {@link CEntryPointErrors}. */ - int initialize(Pointer reservedAddressSpace, UnsignedWord reservedSize, WordPointer basePointer, WordPointer endPointer, NmtVirtualMemoryData nmtData); + int initialize(Pointer reservedAddressSpace, UnsignedWord reservedSize, WordPointer basePointer, WordPointer endPointer); /** * Disposes an instance of the image heap that was created with this provider. This method must diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/os/OSCommittedMemoryProvider.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/os/OSCommittedMemoryProvider.java index 14cb24ae1cde..b09b8bddcd04 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/os/OSCommittedMemoryProvider.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/os/OSCommittedMemoryProvider.java @@ -41,7 +41,6 @@ import com.oracle.svm.core.c.function.CEntryPointErrors; import com.oracle.svm.core.feature.AutomaticallyRegisteredFeature; import com.oracle.svm.core.feature.InternalFeature; -import com.oracle.svm.core.nmt.NmtVirtualMemoryData; public class OSCommittedMemoryProvider extends AbstractCommittedMemoryProvider { @Platforms(Platform.HOSTED_ONLY.class) @@ -50,7 +49,7 @@ public OSCommittedMemoryProvider() { @Override @Uninterruptible(reason = "Still being initialized.") - public int initialize(WordPointer heapBasePointer, CEntryPointCreateIsolateParameters parameters, NmtVirtualMemoryData nmtData) { + public int initialize(WordPointer heapBasePointer, CEntryPointCreateIsolateParameters parameters) { if (!SubstrateOptions.SpawnIsolates.getValue()) { int result = protectSingleIsolateImageHeap(); if (result == CEntryPointErrors.NO_ERROR) { @@ -58,7 +57,7 @@ public int initialize(WordPointer heapBasePointer, CEntryPointCreateIsolateParam } return result; } - return ImageHeapProvider.get().initialize(nullPointer(), zero(), heapBasePointer, nullPointer(), nmtData); + return ImageHeapProvider.get().initialize(nullPointer(), zero(), heapBasePointer, nullPointer()); } @Override diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/os/VirtualMemoryProvider.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/os/VirtualMemoryProvider.java index b30edbda9583..7aa211c1bd41 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/os/VirtualMemoryProvider.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/os/VirtualMemoryProvider.java @@ -31,7 +31,6 @@ import org.graalvm.word.UnsignedWord; import org.graalvm.word.WordBase; import org.graalvm.word.WordFactory; -import com.oracle.svm.core.nmt.NmtVirtualMemoryData; /** * Primitive operations for low-level virtual memory management. @@ -94,8 +93,6 @@ default UnsignedWord getAlignment() { * address range, or {@link WordFactory#nullPointer()} in case of an error. */ Pointer reserve(UnsignedWord nbytes, UnsignedWord alignment, boolean code); - Pointer reserve(UnsignedWord nbytes, UnsignedWord alignment, boolean code, NmtVirtualMemoryData nmtData); - /** * Map a region of an open file to the specified address range. When {@linkplain Access#WRITE @@ -118,8 +115,6 @@ default UnsignedWord getAlignment() { * of an error. */ Pointer mapFile(PointerBase start, UnsignedWord nbytes, WordBase fileHandle, UnsignedWord offset, int access); - Pointer mapFile(PointerBase start, UnsignedWord nbytes, WordBase fileHandle, UnsignedWord offset, int access, NmtVirtualMemoryData nmtData); - /** * Commit an address range so that physical memory or swap memory can be provisioned for it, and @@ -148,7 +143,6 @@ default UnsignedWord getAlignment() { * case of an error, such as inadequate physical memory. */ Pointer commit(PointerBase start, UnsignedWord nbytes, int access); - Pointer commit(PointerBase start, UnsignedWord nbytes, int access, NmtVirtualMemoryData nmtData); /** * Change the protection of a committed address range, or of a subrange of a committed address diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/VMThreads.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/VMThreads.java index f7557013a637..66cb3663e655 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/VMThreads.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/VMThreads.java @@ -56,6 +56,7 @@ import com.oracle.svm.core.locks.VMCondition; import com.oracle.svm.core.locks.VMMutex; import com.oracle.svm.core.log.Log; +import com.oracle.svm.core.nmt.NmtFlag; import com.oracle.svm.core.nodes.CFunctionEpilogueNode; import com.oracle.svm.core.nodes.CFunctionPrologueNode; import com.oracle.svm.core.threadlocal.FastThreadLocal; @@ -240,7 +241,7 @@ public IsolateThread allocateIsolateThread(int isolateThreadSize) { UnsignedWord alignment = WordFactory.unsigned(64); UnsignedWord memorySize = WordFactory.unsigned(isolateThreadSize).add(alignment); - Pointer memory = ImageSingletons.lookup(UnmanagedMemorySupport.class).calloc(memorySize); + Pointer memory = ImageSingletons.lookup(UnmanagedMemorySupport.class).calloc(memorySize, NmtFlag.mtThread.ordinal()); if (memory.isNull()) { return WordFactory.nullPointer(); } diff --git a/substratevm/src/com.oracle.svm.test/src/META-INF/native-image/com.oracle.svm.test/native-image.properties b/substratevm/src/com.oracle.svm.test/src/META-INF/native-image/com.oracle.svm.test/native-image.properties index e1470477f070..c72a60578e4a 100644 --- a/substratevm/src/com.oracle.svm.test/src/META-INF/native-image/com.oracle.svm.test/native-image.properties +++ b/substratevm/src/com.oracle.svm.test/src/META-INF/native-image/com.oracle.svm.test/native-image.properties @@ -6,7 +6,8 @@ Args= \ --features=com.oracle.svm.test.NoProviderConstructorServiceLoaderTest$TestFeature \ --features=com.oracle.svm.test.NativeImageResourceUtils$TestFeature \ --features=com.oracle.svm.test.jfr.JfrTestFeature \ + --features=com.oracle.svm.core.nmt.NmtTestFeature \ --add-opens=java.base/java.lang=ALL-UNNAMED \ --add-exports=org.graalvm.nativeimage.base/com.oracle.svm.util=ALL-UNNAMED \ - --enable-monitoring=jvmstat,jfr,jmxserver,jmxclient,nmt \ + --enable-monitoring=jvmstat,jfr,jmxserver,jmxclient \ -J--enable-preview \ No newline at end of file diff --git a/substratevm/src/com.oracle.svm.test/src/com/oracle/svm/test/nmt/TestBasic.java b/substratevm/src/com.oracle.svm.test/src/com/oracle/svm/test/nmt/TestBasic.java new file mode 100644 index 000000000000..4ae6149fe5cd --- /dev/null +++ b/substratevm/src/com.oracle.svm.test/src/com/oracle/svm/test/nmt/TestBasic.java @@ -0,0 +1,122 @@ +/* + * Copyright (c) 2023, 2023, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2023, 2023, Red Hat Inc. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +package com.oracle.svm.test.nmt; + +import com.oracle.svm.core.nmt.NmtFlag; + +import org.junit.BeforeClass; +import org.junit.Test; + +import com.oracle.svm.core.nmt.NativeMemoryTracking; +import org.graalvm.nativeimage.ImageSingletons; +import org.graalvm.nativeimage.impl.UnmanagedMemorySupport; +import org.graalvm.word.WordFactory; +import org.graalvm.word.Pointer; +import static org.junit.Assert.assertTrue; + +public class TestBasic { + private static final int ALLOCATION_SIZE = 1024 * 16; + private static final int RESERVE_SIZE = ALLOCATION_SIZE; + private static final int REALLOC_SIZE = ALLOCATION_SIZE / 2; + private static final int COMMIT_SIZE = RESERVE_SIZE / 2; + + /** + * This both initializes NMT and does some basic checks to verify pre-init allocations are + * handled. + */ + @BeforeClass + public static void setup() { + // Allocate some memory and check it's being tracked. + Pointer ptr = ImageSingletons.lookup(UnmanagedMemorySupport.class).malloc(WordFactory.unsigned(ALLOCATION_SIZE), NmtFlag.mtTest.ordinal()); + assertTrue(NativeMemoryTracking.getMallocByCategory(NmtFlag.mtTest) == ALLOCATION_SIZE); + + // Realloc previously allocated memory and check NMT has tracked it correctly + Pointer reallocPtr = ImageSingletons.lookup(UnmanagedMemorySupport.class).realloc(ptr, WordFactory.unsigned(REALLOC_SIZE), NmtFlag.mtTest.ordinal()); + assertTrue(NativeMemoryTracking.getMallocByCategory(NmtFlag.mtTest) == REALLOC_SIZE); + + // Free the memory and ensure the tracking is now zeroed. + ImageSingletons.lookup(UnmanagedMemorySupport.class).free(reallocPtr); + assertTrue(NativeMemoryTracking.getMallocByCategory(NmtFlag.mtTest) == 0); + + // Allocate a new block that will live across the initialization boundary + ptr = ImageSingletons.lookup(UnmanagedMemorySupport.class).malloc(WordFactory.unsigned(ALLOCATION_SIZE), NmtFlag.mtTest.ordinal()); + + // We must initialize NMT here so that other tests can use it. + NativeMemoryTracking.initialize(true); + + // Reallocate a block that has lived across the initialization boundary. Verify its been + // recorded. + reallocPtr = ImageSingletons.lookup(UnmanagedMemorySupport.class).realloc(ptr, WordFactory.unsigned(REALLOC_SIZE), NmtFlag.mtTest.ordinal()); + assertTrue(NativeMemoryTracking.getMallocByCategory(NmtFlag.mtTest) == REALLOC_SIZE); + + // Free the memory and verify we're back at zero. + ImageSingletons.lookup(UnmanagedMemorySupport.class).free(reallocPtr); + assertTrue(NativeMemoryTracking.getMallocByCategory(NmtFlag.mtTest) == 0); + } + + @Test + public void testMalloc() throws Throwable { + assertTrue("Test should start with no memory already allocated in the mtTest category.", NativeMemoryTracking.getMallocByCategory(NmtFlag.mtTest) == 0); + + Pointer ptr = ImageSingletons.lookup(UnmanagedMemorySupport.class).malloc(WordFactory.unsigned(ALLOCATION_SIZE), NmtFlag.mtTest.ordinal()); + assertTrue(NativeMemoryTracking.getMallocByCategory(NmtFlag.mtTest) == ALLOCATION_SIZE); + assertTrue(NativeMemoryTracking.getMallocByCategory(NmtFlag.mtNMT) > 0); + + ImageSingletons.lookup(UnmanagedMemorySupport.class).free(ptr); + + assertTrue("After freeing memory for test, mtTest category should have size 0.", NativeMemoryTracking.getMallocByCategory(NmtFlag.mtTest) == 0); + } + + @Test + public void testCalloc() throws Throwable { + assertTrue("Test should start with no memory already allocated in the mtTest category.", NativeMemoryTracking.getMallocByCategory(NmtFlag.mtTest) == 0); + Pointer ptr = ImageSingletons.lookup(UnmanagedMemorySupport.class).calloc(WordFactory.unsigned(ALLOCATION_SIZE), NmtFlag.mtTest.ordinal()); + + assertTrue(NativeMemoryTracking.getMallocByCategory(NmtFlag.mtTest) == ALLOCATION_SIZE); + assertTrue(NativeMemoryTracking.getMallocByCategory(NmtFlag.mtNMT) > 0); + + ImageSingletons.lookup(UnmanagedMemorySupport.class).free(ptr); + + assertTrue("After freeing memory for test, mtTest category should have size 0.", NativeMemoryTracking.getMallocByCategory(NmtFlag.mtTest) == 0); + } + + @Test + public void testRealloc() throws Throwable { + assertTrue("Test should start with no memory already allocated in the mtTest category.", NativeMemoryTracking.getMallocByCategory(NmtFlag.mtTest) == 0); + Pointer ptr = ImageSingletons.lookup(UnmanagedMemorySupport.class).malloc(WordFactory.unsigned(ALLOCATION_SIZE), NmtFlag.mtTest.ordinal()); + + assertTrue(NativeMemoryTracking.getMallocByCategory(NmtFlag.mtTest) == ALLOCATION_SIZE); + assertTrue(NativeMemoryTracking.getMallocByCategory(NmtFlag.mtNMT) > 0); + + Pointer reallocPtr = ImageSingletons.lookup(UnmanagedMemorySupport.class).realloc(ptr, WordFactory.unsigned(REALLOC_SIZE), NmtFlag.mtTest.ordinal()); + + assertTrue(NativeMemoryTracking.getMallocByCategory(NmtFlag.mtTest) == REALLOC_SIZE); + + ImageSingletons.lookup(UnmanagedMemorySupport.class).free(reallocPtr); + assertTrue("After freeing memory for test, mtTest category should have size 0.", NativeMemoryTracking.getMallocByCategory(NmtFlag.mtTest) == 0); + } +} From ac84b8e100b93605bec96e956d307e0362c7a149 Mon Sep 17 00:00:00 2001 From: Robert Toyonaga Date: Mon, 4 Dec 2023 10:46:02 -0500 Subject: [PATCH 4/7] Override and javadoc --- .../graalvm/nativeimage/impl/UnmanagedMemorySupport.java | 4 ++++ .../src/com/oracle/svm/core/nmt/PreInitTable.java | 6 ++++++ 2 files changed, 10 insertions(+) diff --git a/sdk/src/org.graalvm.nativeimage/src/org/graalvm/nativeimage/impl/UnmanagedMemorySupport.java b/sdk/src/org.graalvm.nativeimage/src/org/graalvm/nativeimage/impl/UnmanagedMemorySupport.java index 3bd299b7118c..4dca2cabd080 100644 --- a/sdk/src/org.graalvm.nativeimage/src/org/graalvm/nativeimage/impl/UnmanagedMemorySupport.java +++ b/sdk/src/org.graalvm.nativeimage/src/org/graalvm/nativeimage/impl/UnmanagedMemorySupport.java @@ -46,14 +46,18 @@ /** Implemented by operating-system specific code. */ public interface UnmanagedMemorySupport { T malloc(UnsignedWord size); + T malloc(UnsignedWord size, int flag); T calloc(UnsignedWord size); + T calloc(UnsignedWord size, int flag); T realloc(T ptr, UnsignedWord size); + T realloc(T ptr, UnsignedWord size, int flag); void free(PointerBase ptr); + void untrackedFree(PointerBase ptr); } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/PreInitTable.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/PreInitTable.java index e5d61f036e0b..8b4d909cc8a5 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/PreInitTable.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/PreInitTable.java @@ -59,6 +59,7 @@ protected boolean isEqual(UninterruptibleEntry a, UninterruptibleEntry b) { return a.getHash() == b.getHash(); } + /** This override is necessary to use LibC directly. */ @Override @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) protected UninterruptibleEntry copyToHeap(UninterruptibleEntry valueOnStack) { @@ -72,6 +73,11 @@ protected UninterruptibleEntry copyToHeap(UninterruptibleEntry valueOnStack) { return WordFactory.nullPointer(); } + /** + * Since these native memory hashtable nodes don't have headers, this is necessary to avoid + * re-entering NMT code. Use LibC directly. + */ + @Override @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) protected void free(UninterruptibleEntry entry) { size--; From 26aa93513d33259d97be92ede15e5def0fcded90 Mon Sep 17 00:00:00 2001 From: Christian Haeubl Date: Tue, 6 Feb 2024 18:40:48 +0100 Subject: [PATCH 5/7] Simplifications and cleanups. Pass a native memory tracking category to all relevant allocations of native memory. --- .../graalvm/nativeimage/UnmanagedMemory.java | 47 +- .../impl/UnmanagedMemorySupport.java | 12 +- substratevm/CHANGELOG.md | 1 + .../svm/agent/BreakpointInterceptor.java | 7 +- .../posix/PosixProcessPropertiesSupport.java | 12 +- .../posix/PosixRawFileOperationSupport.java | 4 +- .../posix/UnmanagedMemorySupportImpl.java | 135 ----- .../darwin/DarwinSystemPropertiesSupport.java | 6 +- .../jvmstat/PosixPerfMemoryProvider.java | 11 +- .../posix/thread/PosixPlatformThreads.java | 9 +- .../WindowsSystemPropertiesSupport.java | 8 +- .../com/oracle/svm/core/SubstrateOptions.java | 3 - .../oracle/svm/core/VMInspectionOptions.java | 20 +- .../oracle/svm/core/c/NonmovableArrays.java | 34 +- .../svm/core/c/UnmanagedPrimitiveArrays.java | 19 +- .../core/c/function/IsolateSupportImpl.java | 7 +- .../oracle/svm/core/code/CodeInfoEncoder.java | 15 +- .../DeoptimizationSourcePositionEncoder.java | 16 +- .../svm/core/code/FrameInfoEncoder.java | 3 +- .../oracle/svm/core/code/ImageCodeInfo.java | 3 +- .../code/InstalledCodeObserverSupport.java | 8 +- .../core/code/InstantReferenceAdjuster.java | 5 +- .../svm/core/code/ReferenceAdjuster.java | 9 +- .../svm/core/code/RuntimeCodeCache.java | 3 +- .../svm/core/code/RuntimeCodeInfoAccess.java | 12 +- .../svm/core/code/RuntimeCodeInfoMemory.java | 5 +- .../AbstractUninterruptibleHashtable.java | 23 +- .../collections/GrowableWordArrayAccess.java | 21 +- .../src/com/oracle/svm/core/headers/LibC.java | 4 +- .../oracle/svm/core/headers/LibCSupport.java | 6 + .../svm/core/heap/ReferenceMapEncoder.java | 9 +- .../svm/core/heap/dump/HeapDumpMetadata.java | 18 +- .../core/heap/dump/HeapDumpSupportImpl.java | 8 +- .../svm/core/heap/dump/HeapDumpWriter.java | 5 +- .../svm/core/jdk/SunMiscSubstitutions.java | 13 +- .../svm/core/jdk/TimeZoneSubstitutions.java | 9 +- .../oracle/svm/core/jfr/JfrBufferAccess.java | 12 +- .../oracle/svm/core/jfr/JfrBufferList.java | 4 +- .../svm/core/jfr/JfrBufferNodeAccess.java | 9 +- .../svm/core/jfr/JfrStackTraceRepository.java | 16 +- .../svm/core/jfr/JfrSymbolRepository.java | 11 +- .../svm/core/jfr/utils/JfrVisitedTable.java | 7 + .../oracle/svm/core/jni/JNIJavaVMList.java | 10 +- .../jni/functions/JNIInvocationInterface.java | 10 +- .../core/jvmstat/CHeapPerfMemoryProvider.java | 7 +- .../oracle/svm/core/jvmstat/PerfMemory.java | 10 +- .../oracle/svm/core/memory/NativeMemory.java | 148 ++++++ .../svm/core/memory/NullableNativeMemory.java | 184 +++++++ .../memory/UnmanagedMemorySupportImpl.java} | 56 +- .../memory/UntrackedNullableNativeMemory.java | 135 +++++ .../oracle/svm/core/nmt/HasNmtSupport.java | 43 -- .../svm/core/nmt/NativeMemoryTracking.java | 501 +++--------------- .../nmt/{NmtFlag.java => NmtCategory.java} | 43 +- .../com/oracle/svm/core/nmt/NmtFeature.java | 14 +- ...MallocHeader.java => NmtMallocHeader.java} | 23 +- ...moryInfo.java => NmtMallocMemoryInfo.java} | 30 +- ...shot.java => NmtMallocMemorySnapshot.java} | 32 +- .../oracle/svm/core/nmt/NmtTestFeature.java | 43 -- .../com/oracle/svm/core/nmt/PreInitTable.java | 108 ---- .../oracle/svm/core/nmt/ReturnAddress.java | 42 -- .../os/AbstractRawFileOperationSupport.java | 5 +- .../core/os/BufferedFileOperationSupport.java | 14 +- .../svm/core/os/RawFileOperationSupport.java | 7 +- .../svm/core/sampler/SamplerBufferPool.java | 7 +- .../svm/core/thread/PlatformThreads.java | 9 +- .../com/oracle/svm/core/thread/VMThreads.java | 7 +- .../isolated/IsolatedReferenceAdjuster.java | 20 +- .../IsolatedRuntimeCodeInstaller.java | 16 +- .../IsolatedRuntimeMethodInfoAccess.java | 44 +- .../native-image.properties | 3 +- .../oracle/svm/test/debug/CStructTests.java | 13 +- .../test/nmt/NativeMemoryTrackingTests.java | 90 ++++ .../com/oracle/svm/test/nmt/TestBasic.java | 122 ----- 73 files changed, 1096 insertions(+), 1289 deletions(-) delete mode 100644 substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/UnmanagedMemorySupportImpl.java create mode 100644 substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/memory/NativeMemory.java create mode 100644 substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/memory/NullableNativeMemory.java rename substratevm/src/{com.oracle.svm.core.windows/src/com/oracle/svm/core/windows/WindowsUnmanagedMemorySupportImpl.java => com.oracle.svm.core/src/com/oracle/svm/core/memory/UnmanagedMemorySupportImpl.java} (68%) create mode 100644 substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/memory/UntrackedNullableNativeMemory.java delete mode 100644 substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/HasNmtSupport.java rename substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/{NmtFlag.java => NmtCategory.java} (65%) rename substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/{MallocHeader.java => NmtMallocHeader.java} (72%) rename substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/{MallocMemoryInfo.java => NmtMallocMemoryInfo.java} (81%) rename substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/{MallocMemorySnapshot.java => NmtMallocMemorySnapshot.java} (70%) delete mode 100644 substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/NmtTestFeature.java delete mode 100644 substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/PreInitTable.java delete mode 100644 substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/ReturnAddress.java create mode 100644 substratevm/src/com.oracle.svm.test/src/com/oracle/svm/test/nmt/NativeMemoryTrackingTests.java delete mode 100644 substratevm/src/com.oracle.svm.test/src/com/oracle/svm/test/nmt/TestBasic.java diff --git a/sdk/src/org.graalvm.nativeimage/src/org/graalvm/nativeimage/UnmanagedMemory.java b/sdk/src/org.graalvm.nativeimage/src/org/graalvm/nativeimage/UnmanagedMemory.java index 09a3509e8c0d..3551cf7f486a 100644 --- a/sdk/src/org.graalvm.nativeimage/src/org/graalvm/nativeimage/UnmanagedMemory.java +++ b/sdk/src/org.graalvm.nativeimage/src/org/graalvm/nativeimage/UnmanagedMemory.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016, 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2016, 2024, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * The Universal Permissive License (UPL), Version 1.0 @@ -62,8 +62,7 @@ private UnmanagedMemory() { /** * Allocates {@code size} bytes of unmanaged memory. The content of the memory is undefined. *

- * If {@code size} is 0, the method is allowed but not required to return the null pointer. This - * method never returns a the null pointer, but instead throws a {@link OutOfMemoryError} when + * This method never returns a null pointer, but instead throws an {@link OutOfMemoryError} when * allocation fails. * * @since 19.0 @@ -76,19 +75,10 @@ public static T malloc(UnsignedWord size) { return result; } - public static T malloc(UnsignedWord size, int flag) { - T result = ImageSingletons.lookup(UnmanagedMemorySupport.class).malloc(size, flag); - if (result.isNull()) { - throw new OutOfMemoryError("malloc of unmanaged memory"); - } - return result; - } - /** * Allocates {@code size} bytes of unmanaged memory. The content of the memory is undefined. *

- * If {@code size} is 0, the method is allowed but not required to return the null pointer. This - * method never returns a the null pointer, but instead throws a {@link OutOfMemoryError} when + * This method never returns a null pointer, but instead throws an {@link OutOfMemoryError} when * allocation fails. * * @since 19.0 @@ -100,8 +90,7 @@ public static T malloc(int size) { /** * Allocates {@code size} bytes of unmanaged memory. The content of the memory is set to 0. *

- * If {@code size} is 0, the method is allowed but not required to return the null pointer. This - * method never returns a the null pointer, but instead throws a {@link OutOfMemoryError} when + * This method never returns a null pointer, but instead throws an {@link OutOfMemoryError} when * allocation fails. * * @since 19.0 @@ -117,8 +106,7 @@ public static T calloc(UnsignedWord size) { /** * Allocates {@code size} bytes of unmanaged memory. The content of the memory is set to 0. *

- * If {@code size} is 0, the method is allowed but not required to return the null pointer. This - * method never returns a the null pointer, but instead throws a {@link OutOfMemoryError} when + * This method never returns a null pointer, but instead throws an {@link OutOfMemoryError} when * allocation fails. * * @since 19.0 @@ -132,9 +120,8 @@ public static T calloc(int size) { * If the new size is larger than the old size, the content of the additional memory is * undefined. *

- * If {@code size} is 0, the method is allowed but not required to return the null pointer. This - * method never returns a the null pointer, but instead throws a {@link OutOfMemoryError} when - * allocation fails. + * This method never returns a null pointer, but instead throws an {@link OutOfMemoryError} when + * allocation fails. In that case, the old data is not deallocated and remains unchanged. * * @since 19.0 */ @@ -146,29 +133,13 @@ public static T realloc(T ptr, UnsignedWord size) { return result; } - public static T realloc(T ptr, UnsignedWord size, int flag) { - T result = ImageSingletons.lookup(UnmanagedMemorySupport.class).realloc(ptr, size, flag); - if (result.isNull()) { - throw new OutOfMemoryError("realloc of unmanaged memory"); - } - return result; - } - /** - * Frees unmanaged memory that was previously allocated using methods of this class. + * Frees unmanaged memory that was previously allocated using methods of this class. This method + * is a no-op if the given pointer is {@code null}. * * @since 19.0 */ public static void free(PointerBase ptr) { ImageSingletons.lookup(UnmanagedMemorySupport.class).free(ptr); } - - /** - * Will not attempt to perform any NMT operations. This is crucial for releasing memory - * allocated by C libraries which will not have NMT "malloc headers". If - * {@link UnmanagedMemory#free(PointerBase)} is used instead, a segfault will occur. - */ - public static void untrackedFree(PointerBase ptr) { - ImageSingletons.lookup(UnmanagedMemorySupport.class).untrackedFree(ptr); - } } diff --git a/sdk/src/org.graalvm.nativeimage/src/org/graalvm/nativeimage/impl/UnmanagedMemorySupport.java b/sdk/src/org.graalvm.nativeimage/src/org/graalvm/nativeimage/impl/UnmanagedMemorySupport.java index 4dca2cabd080..dc013c8aa826 100644 --- a/sdk/src/org.graalvm.nativeimage/src/org/graalvm/nativeimage/impl/UnmanagedMemorySupport.java +++ b/sdk/src/org.graalvm.nativeimage/src/org/graalvm/nativeimage/impl/UnmanagedMemorySupport.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017, 2018, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2017, 2024, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * The Universal Permissive License (UPL), Version 1.0 @@ -43,21 +43,13 @@ import org.graalvm.word.PointerBase; import org.graalvm.word.UnsignedWord; -/** Implemented by operating-system specific code. */ +/** Implemented by platform or operating-system specific code. */ public interface UnmanagedMemorySupport { T malloc(UnsignedWord size); - T malloc(UnsignedWord size, int flag); - T calloc(UnsignedWord size); - T calloc(UnsignedWord size, int flag); - T realloc(T ptr, UnsignedWord size); - T realloc(T ptr, UnsignedWord size, int flag); - void free(PointerBase ptr); - - void untrackedFree(PointerBase ptr); } diff --git a/substratevm/CHANGELOG.md b/substratevm/CHANGELOG.md index 0d9063051032..db6620363624 100644 --- a/substratevm/CHANGELOG.md +++ b/substratevm/CHANGELOG.md @@ -6,6 +6,7 @@ This changelog summarizes major changes to GraalVM Native Image. * (GR-51106) Fields that are accessed via a `VarHandle` or `MethodHandle` are no longer marked as "unsafe accessed" when the `VarHandle`/`MethodHandle` can be fully intrinsified. * (GR-49996) Ensure explicitly set image name (e.g., via `-o imagename`) is not accidentally overwritten by `-jar jarfile` option. * (GR-48683) Together with Red Hat, we added partial support for the JFR event `OldObjectSample`. +* (GR-51851) Together with Red Hat, we added initial support for native memory tracking (`--enable-monitoring=nmt`). ## GraalVM for JDK 22 (Internal Version 24.0.0) * (GR-48304) Red Hat added support for the JFR event ThreadAllocationStatistics. diff --git a/substratevm/src/com.oracle.svm.agent/src/com/oracle/svm/agent/BreakpointInterceptor.java b/substratevm/src/com.oracle.svm.agent/src/com/oracle/svm/agent/BreakpointInterceptor.java index ee11c2f4a8b3..5e72875f8fa1 100644 --- a/substratevm/src/com.oracle.svm.agent/src/com/oracle/svm/agent/BreakpointInterceptor.java +++ b/substratevm/src/com.oracle.svm.agent/src/com/oracle/svm/agent/BreakpointInterceptor.java @@ -61,9 +61,6 @@ import java.util.concurrent.locks.ReentrantLock; import java.util.function.Supplier; -import com.oracle.svm.core.jni.headers.JNIMode; -import jdk.graal.compiler.core.common.NumUtil; -import jdk.graal.compiler.java.LambdaUtils; import org.graalvm.nativeimage.StackValue; import org.graalvm.nativeimage.UnmanagedMemory; import org.graalvm.nativeimage.c.function.CEntryPoint; @@ -88,6 +85,7 @@ import com.oracle.svm.core.jni.headers.JNIEnvironment; import com.oracle.svm.core.jni.headers.JNIFieldId; import com.oracle.svm.core.jni.headers.JNIMethodId; +import com.oracle.svm.core.jni.headers.JNIMode; import com.oracle.svm.core.jni.headers.JNINativeMethod; import com.oracle.svm.core.jni.headers.JNIObjectHandle; import com.oracle.svm.core.jni.headers.JNIValue; @@ -106,6 +104,9 @@ import com.oracle.svm.jvmtiagentbase.jvmti.JvmtiInterface; import com.oracle.svm.jvmtiagentbase.jvmti.JvmtiLocationFormat; +import jdk.graal.compiler.core.common.NumUtil; +import jdk.graal.compiler.java.LambdaUtils; + /** * Intercepts events of interest via breakpoints in Java code. *

diff --git a/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/PosixProcessPropertiesSupport.java b/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/PosixProcessPropertiesSupport.java index dfe66b33245d..0a8ef5e06db5 100644 --- a/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/PosixProcessPropertiesSupport.java +++ b/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/PosixProcessPropertiesSupport.java @@ -36,12 +36,12 @@ import org.graalvm.nativeimage.c.type.CCharPointer; import org.graalvm.nativeimage.c.type.CTypeConversion; import org.graalvm.nativeimage.c.type.CTypeConversion.CCharPointerHolder; -import org.graalvm.nativeimage.UnmanagedMemory; import org.graalvm.word.PointerBase; import org.graalvm.word.WordFactory; import com.oracle.svm.core.BaseProcessPropertiesSupport; import com.oracle.svm.core.graal.stackvalue.UnsafeStackValue; +import com.oracle.svm.core.memory.UntrackedNullableNativeMemory; import com.oracle.svm.core.posix.headers.Dlfcn; import com.oracle.svm.core.posix.headers.Signal; import com.oracle.svm.core.posix.headers.Stdlib; @@ -103,7 +103,7 @@ public String getObjectFile(PointerBase symbolAddress) { try { return CTypeConversion.toJavaString(realpath); } finally { - UnmanagedMemory.untrackedFree(realpath); + UntrackedNullableNativeMemory.free(realpath); } } @@ -157,14 +157,14 @@ protected static String realpath(String path) { * pointer to it, so I have to free it. */ try (CCharPointerHolder pathHolder = CTypeConversion.toCString(path)) { - final CCharPointer realpathPointer = Stdlib.realpath(pathHolder.get(), WordFactory.nullPointer()); - if (realpathPointer.isNull()) { + CCharPointer realpath = Stdlib.realpath(pathHolder.get(), WordFactory.nullPointer()); + if (realpath.isNull()) { /* Failure to find a real path. */ return null; } else { /* Success */ - final String result = CTypeConversion.toJavaString(realpathPointer); - UnmanagedMemory.untrackedFree(realpathPointer); + String result = CTypeConversion.toJavaString(realpath); + UntrackedNullableNativeMemory.free(realpath); return result; } } diff --git a/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/PosixRawFileOperationSupport.java b/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/PosixRawFileOperationSupport.java index b318b1b93cd9..af5da227d5fd 100644 --- a/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/PosixRawFileOperationSupport.java +++ b/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/PosixRawFileOperationSupport.java @@ -32,7 +32,6 @@ import org.graalvm.nativeimage.Platforms; import org.graalvm.nativeimage.c.type.CCharPointer; import org.graalvm.nativeimage.c.type.CTypeConversion; -import org.graalvm.nativeimage.impl.UnmanagedMemorySupport; import org.graalvm.word.Pointer; import org.graalvm.word.SignedWord; import org.graalvm.word.UnsignedWord; @@ -42,6 +41,7 @@ import com.oracle.svm.core.feature.AutomaticallyRegisteredFeature; import com.oracle.svm.core.feature.InternalFeature; import com.oracle.svm.core.headers.LibC; +import com.oracle.svm.core.memory.UntrackedNullableNativeMemory; import com.oracle.svm.core.os.AbstractRawFileOperationSupport; import com.oracle.svm.core.os.AbstractRawFileOperationSupport.RawFileOperationSupportHolder; import com.oracle.svm.core.posix.headers.Errno; @@ -58,7 +58,7 @@ public PosixRawFileOperationSupport(boolean useNativeByteOrder) { @Override public CCharPointer allocateCPath(String path) { byte[] data = path.getBytes(); - CCharPointer filename = ImageSingletons.lookup(UnmanagedMemorySupport.class).malloc(WordFactory.unsigned(data.length + 1)); + CCharPointer filename = UntrackedNullableNativeMemory.malloc(WordFactory.unsigned(data.length + 1)); if (filename.isNull()) { return WordFactory.nullPointer(); } diff --git a/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/UnmanagedMemorySupportImpl.java b/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/UnmanagedMemorySupportImpl.java deleted file mode 100644 index 8f03ad222df9..000000000000 --- a/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/UnmanagedMemorySupportImpl.java +++ /dev/null @@ -1,135 +0,0 @@ -/* - * Copyright (c) 2016, 2019, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. Oracle designates this - * particular file as subject to the "Classpath" exception as provided - * by Oracle in the LICENSE file that accompanied this code. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - */ -package com.oracle.svm.core.posix; - -import jdk.graal.compiler.api.replacements.Fold; -import org.graalvm.nativeimage.ImageSingletons; -import org.graalvm.nativeimage.impl.UnmanagedMemorySupport; - -import org.graalvm.word.PointerBase; -import org.graalvm.word.Pointer; -import org.graalvm.word.UnsignedWord; -import org.graalvm.word.WordFactory; -import org.graalvm.nativeimage.StackValue; - -import com.oracle.svm.core.Uninterruptible; -import com.oracle.svm.core.feature.AutomaticallyRegisteredImageSingleton; -import com.oracle.svm.core.headers.LibCSupport; -import com.oracle.svm.core.nmt.NmtFlag; -import com.oracle.svm.core.nmt.NativeMemoryTracking; -import com.oracle.svm.core.nmt.ReturnAddress; - -@AutomaticallyRegisteredImageSingleton(UnmanagedMemorySupport.class) -class UnmanagedMemorySupportImpl implements UnmanagedMemorySupport { - @Override - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - public T malloc(UnsignedWord size) { - return malloc(size, NmtFlag.mtNone.ordinal()); - } - - @Override - @SuppressWarnings("unchecked") - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - public T malloc(UnsignedWord size, int flag) { - ReturnAddress ra = StackValue.get(ReturnAddress.class); - if (NativeMemoryTracking.handlePreInitMallocs(size, flag, ra)) { - return (T) ra.get(); - } - Pointer outerPointer = libc().malloc(size.add(NativeMemoryTracking.getHeaderSize())); - return (T) NativeMemoryTracking.recordMalloc(outerPointer, size, flag); - } - - @Override - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - public T calloc(UnsignedWord size) { - return calloc(size, NmtFlag.mtNone.ordinal()); - } - - @Override - @SuppressWarnings("unchecked") - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - public T calloc(UnsignedWord size, int flag) { - ReturnAddress ra = StackValue.get(ReturnAddress.class); - if (NativeMemoryTracking.handlePreInitCallocs(size, flag, ra)) { - return (T) ra.get(); - } - Pointer outerPointer = libc().calloc(WordFactory.unsigned(1), size.add(NativeMemoryTracking.getHeaderSize())); - return (T) NativeMemoryTracking.recordMalloc(outerPointer, size, flag); - } - - @Override - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - public T realloc(T ptr, UnsignedWord size) { - return realloc(ptr, size, NmtFlag.mtNone.ordinal()); - } - - @Override - @SuppressWarnings("unchecked") - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - public T realloc(T ptr, UnsignedWord size, int flag) { - - ReturnAddress ra = StackValue.get(ReturnAddress.class); - if (NativeMemoryTracking.handlePreInitReallocs((Pointer) ptr, size, flag, ra)) { - return (T) ra.get(); - } - - // Retrieve necessary data from the old block - Pointer oldOuterPointer = ((Pointer) ptr).subtract(NativeMemoryTracking.getHeaderSize()); - long oldSize = NativeMemoryTracking.getAllocationSize(oldOuterPointer); - int oldCategory = NativeMemoryTracking.getAllocationCategory(oldOuterPointer); - - // Perform the realloc - Pointer newOuterPointer = libc().realloc(oldOuterPointer, size.add(NativeMemoryTracking.getHeaderSize())); - - // Only deaccount the old block, if we were successful. - if (newOuterPointer.isNonNull()) { - NativeMemoryTracking.deaccountMalloc(oldSize, oldCategory); - } - - // Account the new block and overwrite the header with the new tracking data - return (T) NativeMemoryTracking.recordMalloc(newOuterPointer, size, flag); - } - - @Override - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - public void free(PointerBase ptr) { - if (NativeMemoryTracking.handlePreInitFrees((Pointer) ptr)) { - return; - } - NativeMemoryTracking.deaccountMalloc(ptr); - libc().free(((Pointer) ptr).subtract(NativeMemoryTracking.getHeaderSize())); - } - - @Override - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - public void untrackedFree(PointerBase ptr) { - libc().free(ptr); - } - - @Fold - static LibCSupport libc() { - return ImageSingletons.lookup(LibCSupport.class); - } -} diff --git a/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/darwin/DarwinSystemPropertiesSupport.java b/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/darwin/DarwinSystemPropertiesSupport.java index 88dc4551d9e3..283f30947ae1 100644 --- a/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/darwin/DarwinSystemPropertiesSupport.java +++ b/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/darwin/DarwinSystemPropertiesSupport.java @@ -31,7 +31,6 @@ import org.graalvm.nativeimage.c.type.CTypeConversion; import org.graalvm.nativeimage.c.type.CTypeConversion.CCharPointerHolder; import org.graalvm.nativeimage.impl.RuntimeSystemPropertiesSupport; -import org.graalvm.nativeimage.UnmanagedMemory; import org.graalvm.word.UnsignedWord; import org.graalvm.word.WordFactory; @@ -39,6 +38,7 @@ import com.oracle.svm.core.feature.InternalFeature; import com.oracle.svm.core.graal.stackvalue.UnsafeStackValue; import com.oracle.svm.core.jdk.SystemPropertiesSupport; +import com.oracle.svm.core.memory.UntrackedNullableNativeMemory; import com.oracle.svm.core.posix.PosixSystemPropertiesSupport; import com.oracle.svm.core.posix.headers.Limits; import com.oracle.svm.core.posix.headers.Stdlib; @@ -105,7 +105,7 @@ protected String osVersionValue() { CCharPointer osVersionStr = Foundation.systemVersionPlatform(); if (osVersionStr.isNonNull()) { osVersionValue = CTypeConversion.toJavaString(osVersionStr); - UnmanagedMemory.untrackedFree(osVersionStr); + UntrackedNullableNativeMemory.free(osVersionStr); return osVersionValue; } } else { @@ -120,7 +120,7 @@ protected String osVersionValue() { CCharPointer osVersionStr = Foundation.systemVersionPlatformFallback(); if (osVersionStr.isNonNull()) { osVersionValue = CTypeConversion.toJavaString(osVersionStr); - UnmanagedMemory.untrackedFree(osVersionStr); + UntrackedNullableNativeMemory.free(osVersionStr); return osVersionValue; } return osVersionValue = "Unknown"; diff --git a/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/jvmstat/PosixPerfMemoryProvider.java b/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/jvmstat/PosixPerfMemoryProvider.java index 1942cde8ee0d..fffe6ebe1034 100644 --- a/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/jvmstat/PosixPerfMemoryProvider.java +++ b/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/jvmstat/PosixPerfMemoryProvider.java @@ -41,15 +41,12 @@ import java.nio.ByteBuffer; -import jdk.graal.compiler.core.common.NumUtil; import org.graalvm.nativeimage.ImageSingletons; import org.graalvm.nativeimage.Platform; import org.graalvm.nativeimage.Platforms; import org.graalvm.nativeimage.StackValue; -import org.graalvm.nativeimage.UnmanagedMemory; import org.graalvm.nativeimage.c.type.CCharPointer; import org.graalvm.nativeimage.c.type.CTypeConversion; -import org.graalvm.nativeimage.impl.UnmanagedMemorySupport; import org.graalvm.word.Pointer; import org.graalvm.word.WordFactory; @@ -64,6 +61,8 @@ import com.oracle.svm.core.jvmstat.PerfManager; import com.oracle.svm.core.jvmstat.PerfMemoryPrologue; import com.oracle.svm.core.jvmstat.PerfMemoryProvider; +import com.oracle.svm.core.memory.NullableNativeMemory; +import com.oracle.svm.core.nmt.NmtCategory; import com.oracle.svm.core.os.RawFileOperationSupport; import com.oracle.svm.core.os.RawFileOperationSupport.RawFileDescriptor; import com.oracle.svm.core.os.VirtualMemoryProvider; @@ -80,6 +79,8 @@ import com.oracle.svm.core.posix.headers.Signal; import com.oracle.svm.core.posix.headers.Unistd; +import jdk.graal.compiler.core.common.NumUtil; + /** * This class uses high-level JDK features at the moment. In the future, we will need to rewrite * this code so that it can be executed during the isolate startup (i.e., in uninterruptible code), @@ -157,7 +158,7 @@ private static String getUserName(int uid) { } /* Retrieve the username and copy it to a String object. */ - CCharPointer pwBuf = ImageSingletons.lookup(UnmanagedMemorySupport.class).malloc(WordFactory.unsigned(bufSize)); + CCharPointer pwBuf = NullableNativeMemory.malloc(WordFactory.unsigned(bufSize), NmtCategory.JvmStat); if (pwBuf.isNull()) { return null; } @@ -182,7 +183,7 @@ private static String getUserName(int uid) { return CTypeConversion.toJavaString(pwName); } finally { - UnmanagedMemory.free(pwBuf); + NullableNativeMemory.free(pwBuf); } } diff --git a/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/thread/PosixPlatformThreads.java b/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/thread/PosixPlatformThreads.java index f3b29b1f170f..5dc290d87ecd 100644 --- a/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/thread/PosixPlatformThreads.java +++ b/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/thread/PosixPlatformThreads.java @@ -24,19 +24,16 @@ */ package com.oracle.svm.core.posix.thread; -import org.graalvm.nativeimage.ImageSingletons; import org.graalvm.nativeimage.Platform; import org.graalvm.nativeimage.Platform.HOSTED_ONLY; import org.graalvm.nativeimage.Platforms; import org.graalvm.nativeimage.StackValue; -import org.graalvm.nativeimage.UnmanagedMemory; import org.graalvm.nativeimage.c.function.CFunctionPointer; import org.graalvm.nativeimage.c.struct.SizeOf; import org.graalvm.nativeimage.c.type.CTypeConversion; import org.graalvm.nativeimage.c.type.CTypeConversion.CCharPointerHolder; import org.graalvm.nativeimage.c.type.VoidPointer; import org.graalvm.nativeimage.c.type.WordPointer; -import org.graalvm.nativeimage.impl.UnmanagedMemorySupport; import org.graalvm.word.Pointer; import org.graalvm.word.PointerBase; import org.graalvm.word.UnsignedWord; @@ -49,6 +46,8 @@ import com.oracle.svm.core.annotate.TargetClass; import com.oracle.svm.core.feature.AutomaticallyRegisteredImageSingleton; import com.oracle.svm.core.graal.stackvalue.UnsafeStackValue; +import com.oracle.svm.core.memory.NativeMemory; +import com.oracle.svm.core.nmt.NmtCategory; import com.oracle.svm.core.posix.PosixUtils; import com.oracle.svm.core.posix.headers.Errno; import com.oracle.svm.core.posix.headers.Pthread; @@ -322,7 +321,7 @@ final class PosixParker extends Parker { // Allocate mutex and condition in a single step so that they are adjacent in memory. UnsignedWord mutexSize = SizeOf.unsigned(pthread_mutex_t.class); UnsignedWord condSize = SizeOf.unsigned(pthread_cond_t.class); - Pointer memory = UnmanagedMemory.malloc(mutexSize.add(condSize.multiply(2))); + Pointer memory = NativeMemory.malloc(mutexSize.add(condSize.multiply(2)), NmtCategory.Threading); mutex = (pthread_mutex_t) memory; relativeCond = (pthread_cond_t) memory.add(mutexSize); absoluteCond = (pthread_cond_t) memory.add(mutexSize).add(condSize); @@ -436,7 +435,7 @@ protected void release() { status = Pthread.pthread_mutex_destroy(mutex); assert status == 0; - ImageSingletons.lookup(UnmanagedMemorySupport.class).free(mutex); + NativeMemory.free(mutex); mutex = WordFactory.nullPointer(); } } diff --git a/substratevm/src/com.oracle.svm.core.windows/src/com/oracle/svm/core/windows/WindowsSystemPropertiesSupport.java b/substratevm/src/com.oracle.svm.core.windows/src/com/oracle/svm/core/windows/WindowsSystemPropertiesSupport.java index 06b9509dc89f..5795754098cd 100644 --- a/substratevm/src/com.oracle.svm.core.windows/src/com/oracle/svm/core/windows/WindowsSystemPropertiesSupport.java +++ b/substratevm/src/com.oracle.svm.core.windows/src/com/oracle/svm/core/windows/WindowsSystemPropertiesSupport.java @@ -38,7 +38,6 @@ import org.graalvm.nativeimage.c.type.VoidPointer; import org.graalvm.nativeimage.c.type.WordPointer; import org.graalvm.nativeimage.impl.RuntimeSystemPropertiesSupport; -import org.graalvm.nativeimage.impl.UnmanagedMemorySupport; import org.graalvm.word.UnsignedWord; import org.graalvm.word.WordFactory; @@ -47,6 +46,8 @@ import com.oracle.svm.core.feature.InternalFeature; import com.oracle.svm.core.graal.stackvalue.UnsafeStackValue; import com.oracle.svm.core.jdk.SystemPropertiesSupport; +import com.oracle.svm.core.memory.NullableNativeMemory; +import com.oracle.svm.core.nmt.NmtCategory; import com.oracle.svm.core.util.VMError; import com.oracle.svm.core.windows.headers.FileAPI; import com.oracle.svm.core.windows.headers.LibLoaderAPI; @@ -245,12 +246,11 @@ public Pair getOsNameAndVersion() { break; } - VoidPointer versionInfo = ImageSingletons.lookup(UnmanagedMemorySupport.class).malloc(WordFactory.unsigned(versionSize)); + VoidPointer versionInfo = NullableNativeMemory.malloc(WordFactory.unsigned(versionSize), NmtCategory.Other); if (versionInfo.isNull()) { break; } try { - if (WinVer.GetFileVersionInfoW(kernel32Path, 0, versionSize, versionInfo) == 0) { break; } @@ -267,7 +267,7 @@ public Pair getOsNameAndVersion() { minorVersion = (short) fileInfo.dwProductVersionMS(); // LOWORD buildNumber = (short) (fileInfo.dwProductVersionLS() >> 16); // HIWORD } finally { - ImageSingletons.lookup(UnmanagedMemorySupport.class).free(versionInfo); + NullableNativeMemory.free(versionInfo); } } while (false); diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java index 7c3e71227668..a567daeb4631 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java @@ -923,9 +923,6 @@ public Boolean getValue(OptionValues values) { @Option(help = "file:doc-files/FlightRecorderOptionsHelp.txt")// public static final RuntimeOptionKey FlightRecorderOptions = new RuntimeOptionKey<>("", Immutable); - @Option(help = "Enable native memory tracking")// - public static final RuntimeOptionKey NativeMemoryTracking = new RuntimeOptionKey<>(false); - public static String reportsPath() { Path reportsPath = ImageSingletons.lookup(ReportingSupport.class).reportsPath; if (reportsPath.isAbsolute()) { diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/VMInspectionOptions.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/VMInspectionOptions.java index 9e5f61a768b1..b2f3d0d1e774 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/VMInspectionOptions.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/VMInspectionOptions.java @@ -30,13 +30,10 @@ import java.util.Set; import org.graalvm.collections.EconomicMap; -import jdk.graal.compiler.api.replacements.Fold; -import jdk.graal.compiler.options.Option; -import jdk.graal.compiler.options.OptionKey; -import jdk.graal.compiler.options.OptionType; import org.graalvm.nativeimage.Platform; import org.graalvm.nativeimage.Platform.WINDOWS; import org.graalvm.nativeimage.Platforms; +import org.graalvm.nativeimage.impl.InternalPlatform; import com.oracle.svm.core.heap.dump.HeapDumping; import com.oracle.svm.core.jdk.management.ManagementAgentModule; @@ -47,6 +44,11 @@ import com.oracle.svm.core.util.UserError; import com.oracle.svm.util.LogUtils; +import jdk.graal.compiler.api.replacements.Fold; +import jdk.graal.compiler.options.Option; +import jdk.graal.compiler.options.OptionKey; +import jdk.graal.compiler.options.OptionType; + public final class VMInspectionOptions { private static final String ENABLE_MONITORING_OPTION = "enable-monitoring"; private static final String MONITORING_DEFAULT_NAME = ""; @@ -62,8 +64,8 @@ public final class VMInspectionOptions { private static final List MONITORING_ALL_VALUES = List.of(MONITORING_HEAPDUMP_NAME, MONITORING_JFR_NAME, MONITORING_JVMSTAT_NAME, MONITORING_JMXCLIENT_NAME, MONITORING_JMXSERVER_NAME, MONITORING_THREADDUMP_NAME, MONITORING_NMT_NAME, MONITORING_ALL_NAME, MONITORING_DEFAULT_NAME); private static final String MONITORING_ALLOWED_VALUES_TEXT = "'" + MONITORING_HEAPDUMP_NAME + "', '" + MONITORING_JFR_NAME + "', '" + MONITORING_JVMSTAT_NAME + "', '" + MONITORING_JMXSERVER_NAME + - "' (experimental), '" + MONITORING_JMXCLIENT_NAME + "' (experimental), '" + MONITORING_THREADDUMP_NAME + "', '" + MONITORING_NMT_NAME + "', or '" + MONITORING_ALL_NAME + - "' (deprecated behavior: defaults to '" + MONITORING_ALL_NAME + "' if no argument is provided)"; + "' (experimental), '" + MONITORING_JMXCLIENT_NAME + "' (experimental), '" + MONITORING_THREADDUMP_NAME + "', '" + MONITORING_NMT_NAME + "' (experimental), or '" + + MONITORING_ALL_NAME + "' (deprecated behavior: defaults to '" + MONITORING_ALL_NAME + "' if no argument is provided)"; static { assert MONITORING_ALL_VALUES.stream().allMatch(v -> MONITORING_DEFAULT_NAME.equals(v) || MONITORING_ALLOWED_VALUES_TEXT.contains(v)) : "A value is missing in the user-facing help text"; @@ -165,8 +167,10 @@ public static boolean hasThreadDumpSupport() { } @Fold - public static boolean hasNmtSupport() { - return hasAllOrKeywordMonitoringSupport(MONITORING_NMT_NAME) && !Platform.includedIn(WINDOWS.class); + public static boolean hasNativeMemoryTrackingSupport() { + // TEMP (chaeubl): for testing in the CI. + return Platform.includedIn(InternalPlatform.NATIVE_ONLY.class); + // return hasAllOrKeywordMonitoringSupport(MONITORING_NMT_NAME); } @Option(help = "Dumps all runtime compiled methods on SIGUSR2.", type = OptionType.User) // diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/c/NonmovableArrays.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/c/NonmovableArrays.java index 1b765e534da2..b0390c59a113 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/c/NonmovableArrays.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/c/NonmovableArrays.java @@ -28,11 +28,9 @@ import java.nio.ByteBuffer; import java.util.Arrays; -import org.graalvm.nativeimage.ImageSingletons; import org.graalvm.nativeimage.Platform; import org.graalvm.nativeimage.Platforms; import org.graalvm.nativeimage.c.type.CTypeConversion; -import org.graalvm.nativeimage.impl.UnmanagedMemorySupport; import org.graalvm.word.Pointer; import org.graalvm.word.PointerBase; import org.graalvm.word.UnsignedWord; @@ -51,6 +49,8 @@ import com.oracle.svm.core.hub.DynamicHub; import com.oracle.svm.core.hub.LayoutEncoding; import com.oracle.svm.core.jdk.UninterruptibleUtils; +import com.oracle.svm.core.memory.NullableNativeMemory; +import com.oracle.svm.core.nmt.NmtCategory; import com.oracle.svm.core.snippets.KnownIntrinsics; import com.oracle.svm.core.util.VMError; @@ -84,7 +84,7 @@ public final class NonmovableArrays { @SuppressWarnings("unchecked") @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - private static > T createArray(int length, Class arrayType) { + private static > T createArray(int length, Class arrayType, NmtCategory nmtCategory) { if (SubstrateUtil.HOSTED) { Class componentType = arrayType.getComponentType(); Object array = Array.newInstance(componentType, length); @@ -93,7 +93,7 @@ private static > T createArray(int length, Class DynamicHub hub = SubstrateUtil.cast(arrayType, DynamicHub.class); assert LayoutEncoding.isArray(hub.getLayoutEncoding()); UnsignedWord size = LayoutEncoding.getArrayAllocationSize(hub.getLayoutEncoding(), length); - Pointer array = ImageSingletons.lookup(UnmanagedMemorySupport.class).calloc(size); + Pointer array = NullableNativeMemory.calloc(size, nmtCategory); if (array.isNull()) { throw OUT_OF_MEMORY_ERROR; } @@ -181,8 +181,8 @@ public static > T nullArray() { * array, it is not a Java object and must never be referenced as an object. */ @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - public static NonmovableArray createByteArray(int nbytes) { - return createArray(nbytes, byte[].class); + public static NonmovableArray createByteArray(int nbytes, NmtCategory nmtCategory) { + return createArray(nbytes, byte[].class, nmtCategory); } /** @@ -193,8 +193,8 @@ public static NonmovableArray createByteArray(int nbytes) { * of a Java array, it is not a Java object and must never be referenced as an object. */ @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - public static NonmovableArray createIntArray(int length) { - return createArray(length, int[].class); + public static NonmovableArray createIntArray(int length, NmtCategory nmtCategory) { + return createArray(length, int[].class, nmtCategory); } /** @@ -205,8 +205,8 @@ public static NonmovableArray createIntArray(int length) { * of a Java array, it is not a Java object and must never be referenced as an object. */ @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - public static NonmovableArray createWordArray(int length) { - return createArray(length, WordBase[].class); + public static NonmovableArray createWordArray(int length, NmtCategory nmtCategory) { + return createArray(length, WordBase[].class, nmtCategory); } /** @@ -220,16 +220,16 @@ public static NonmovableArray createWordArray(int length * that of a Java array, it is not a Java object and must never be referenced as an object. */ @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - public static NonmovableObjectArray createObjectArray(Class arrayType, int length) { + public static NonmovableObjectArray createObjectArray(Class arrayType, int length, NmtCategory nmtCategory) { assert (SubstrateUtil.HOSTED ? (arrayType.isArray() && !arrayType.getComponentType().isPrimitive()) : LayoutEncoding.isObjectArray(SubstrateUtil.cast(arrayType, DynamicHub.class).getLayoutEncoding())) : "must be an object array type"; - return createArray(length, arrayType); + return createArray(length, arrayType, nmtCategory); } /** @see java.util.Arrays#copyOf */ @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - public static NonmovableObjectArray copyOfObjectArray(T[] source, int newLength) { - NonmovableObjectArray array = createArray(newLength, source.getClass()); + public static NonmovableObjectArray copyOfObjectArray(T[] source, int newLength, NmtCategory nmtCategory) { + NonmovableObjectArray array = createArray(newLength, source.getClass(), nmtCategory); int copyLength = (source.length < newLength) ? source.length : newLength; for (int i = 0; i < copyLength; i++) { setObject(array, i, source[i]); @@ -239,8 +239,8 @@ public static NonmovableObjectArray copyOfObjectArray(T[] source, int new /** Same as {@link #copyOfObjectArray} with a {@code newLength} of the array length. */ @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - public static NonmovableObjectArray copyOfObjectArray(T[] source) { - return copyOfObjectArray(source, source.length); + public static NonmovableObjectArray copyOfObjectArray(T[] source, NmtCategory nmtCategory) { + return copyOfObjectArray(source, source.length, nmtCategory); } public static byte[] heapCopyOfByteArray(NonmovableArray source) { @@ -299,7 +299,7 @@ private static T arraycopyToHeap(NonmovableArray src, int srcPos, T dest, /** Releases an array created at runtime. */ @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static void releaseUnmanagedArray(NonmovableArray array) { - ImageSingletons.lookup(UnmanagedMemorySupport.class).free(array); + NullableNativeMemory.free(array); untrackUnmanagedArray(array); } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/c/UnmanagedPrimitiveArrays.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/c/UnmanagedPrimitiveArrays.java index a23112ad2cce..251b914c27ec 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/c/UnmanagedPrimitiveArrays.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/c/UnmanagedPrimitiveArrays.java @@ -24,6 +24,11 @@ */ package com.oracle.svm.core.c; +import org.graalvm.word.Pointer; +import org.graalvm.word.PointerBase; +import org.graalvm.word.UnsignedWord; +import org.graalvm.word.WordFactory; + import com.oracle.svm.core.JavaMemoryUtil; import com.oracle.svm.core.SubstrateUtil; import com.oracle.svm.core.Uninterruptible; @@ -32,17 +37,13 @@ import com.oracle.svm.core.hub.DynamicHub; import com.oracle.svm.core.hub.LayoutEncoding; import com.oracle.svm.core.jdk.UninterruptibleUtils; +import com.oracle.svm.core.memory.NullableNativeMemory; +import com.oracle.svm.core.nmt.NmtCategory; import com.oracle.svm.core.snippets.KnownIntrinsics; import com.oracle.svm.core.util.VMError; import jdk.graal.compiler.nodes.java.ArrayLengthNode; import jdk.graal.compiler.word.Word; -import org.graalvm.nativeimage.ImageSingletons; -import org.graalvm.nativeimage.impl.UnmanagedMemorySupport; -import org.graalvm.word.Pointer; -import org.graalvm.word.PointerBase; -import org.graalvm.word.UnsignedWord; -import org.graalvm.word.WordFactory; /** * Support for allocating and accessing primitive element arrays created in unmanaged memory. They @@ -57,11 +58,11 @@ public final class UnmanagedPrimitiveArrays { @SuppressWarnings("unchecked") @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - public static > T createArray(int length, Class arrayType) { + public static > T createArray(int length, Class arrayType, NmtCategory nmtCategory) { DynamicHub hub = SubstrateUtil.cast(arrayType, DynamicHub.class); VMError.guarantee(LayoutEncoding.isPrimitiveArray(hub.getLayoutEncoding())); UnsignedWord size = WordFactory.unsigned(length).shiftLeft(LayoutEncoding.getArrayIndexShift(hub.getLayoutEncoding())); - Pointer array = ImageSingletons.lookup(UnmanagedMemorySupport.class).calloc(size); + Pointer array = NullableNativeMemory.calloc(size, nmtCategory); if (array.isNull()) { throw OUT_OF_MEMORY_ERROR; } @@ -88,7 +89,7 @@ public static void untrackUnmanagedArray(UnmanagedPrimitiveArray array) { @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static void releaseUnmanagedArray(UnmanagedPrimitiveArray array) { untrackUnmanagedArray(array); - ImageSingletons.lookup(UnmanagedMemorySupport.class).free(array); + NullableNativeMemory.free(array); } /** Returns a pointer to the address of the given index of an array. */ diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/c/function/IsolateSupportImpl.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/c/function/IsolateSupportImpl.java index 5a47755a8c91..4571f63a383d 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/c/function/IsolateSupportImpl.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/c/function/IsolateSupportImpl.java @@ -31,7 +31,6 @@ import org.graalvm.nativeimage.Isolates.CreateIsolateParameters; import org.graalvm.nativeimage.Isolates.IsolateException; import org.graalvm.nativeimage.Isolates.ProtectionDomain; -import org.graalvm.nativeimage.UnmanagedMemory; import org.graalvm.nativeimage.c.struct.SizeOf; import org.graalvm.nativeimage.c.type.CCharPointerPointer; import org.graalvm.nativeimage.c.type.CTypeConversion; @@ -42,6 +41,8 @@ import com.oracle.svm.core.c.function.CEntryPointNativeFunctions.IsolateThreadPointer; import com.oracle.svm.core.feature.AutomaticallyRegisteredImageSingleton; import com.oracle.svm.core.graal.stackvalue.UnsafeStackValue; +import com.oracle.svm.core.memory.NativeMemory; +import com.oracle.svm.core.nmt.NmtCategory; import com.oracle.svm.core.option.SubstrateOptionsParser; import com.oracle.svm.core.os.MemoryProtectionProvider; import com.oracle.svm.core.os.MemoryProtectionProvider.UnsupportedDomainException; @@ -89,7 +90,7 @@ public static IsolateThread createIsolate(CreateIsolateParameters parameters, bo // Internally, we use C-style arguments, i.e., the first argument is reserved for // the name of the binary. We use null when isolates are created manually. argc = isolateArgCount + 1; - argv = UnmanagedMemory.malloc(SizeOf.unsigned(CCharPointerPointer.class).multiply(argc)); + argv = NativeMemory.malloc(SizeOf.unsigned(CCharPointerPointer.class).multiply(argc), NmtCategory.Other); argv.write(0, WordFactory.nullPointer()); pointerHolders = new CTypeConversion.CCharPointerHolder[isolateArgCount]; @@ -121,7 +122,7 @@ public static IsolateThread createIsolate(CreateIsolateParameters parameters, bo for (CTypeConversion.CCharPointerHolder ph : pointerHolders) { ph.close(); } - UnmanagedMemory.free(params.getArgv()); + NativeMemory.free(params.getArgv()); } throwOnError(result); diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/CodeInfoEncoder.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/CodeInfoEncoder.java index c58ab2d74eda..3d3cfb28896e 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/CodeInfoEncoder.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/CodeInfoEncoder.java @@ -58,6 +58,7 @@ import com.oracle.svm.core.meta.SharedField; import com.oracle.svm.core.meta.SharedMethod; import com.oracle.svm.core.meta.SharedType; +import com.oracle.svm.core.nmt.NmtCategory; import com.oracle.svm.core.option.HostedOptionKey; import com.oracle.svm.core.util.ByteArrayReader; import com.oracle.svm.core.util.Counter; @@ -129,12 +130,10 @@ private void encodeAllAndInstall(CodeInfo target, ReferenceAdjuster adjuster) { } @Uninterruptible(reason = "Nonmovable object arrays are not visible to GC until installed in target.") - private static void install(CodeInfo target, JavaConstant[] objectConstantsArray, Class[] sourceClassesArray, - String[] sourceMethodNamesArray, ReferenceAdjuster adjuster) { - - NonmovableObjectArray frameInfoObjectConstants = adjuster.copyOfObjectConstantArray(objectConstantsArray); - NonmovableObjectArray> frameInfoSourceClasses = (sourceClassesArray != null) ? adjuster.copyOfObjectArray(sourceClassesArray) : NonmovableArrays.nullArray(); - NonmovableObjectArray frameInfoSourceMethodNames = (sourceMethodNamesArray != null) ? adjuster.copyOfObjectArray(sourceMethodNamesArray) : NonmovableArrays.nullArray(); + private static void install(CodeInfo target, JavaConstant[] objectConstants, Class[] sourceClasses, String[] sourceMethodNames, ReferenceAdjuster adjuster) { + NonmovableObjectArray frameInfoObjectConstants = adjuster.copyOfObjectConstantArray(objectConstants, NmtCategory.Code); + NonmovableObjectArray> frameInfoSourceClasses = (sourceClasses != null) ? adjuster.copyOfObjectArray(sourceClasses, NmtCategory.Code) : NonmovableArrays.nullArray(); + NonmovableObjectArray frameInfoSourceMethodNames = (sourceMethodNames != null) ? adjuster.copyOfObjectArray(sourceMethodNames, NmtCategory.Code) : NonmovableArrays.nullArray(); CodeInfoAccess.setEncodings(target, frameInfoObjectConstants, frameInfoSourceClasses, frameInfoSourceMethodNames); } @@ -339,9 +338,9 @@ private void encodeIPData() { writeEncodedFrameInfo(encodingBuffer, data, entryFlags); } - codeInfoIndex = NonmovableArrays.createByteArray(TypeConversion.asU4(indexBuffer.getBytesWritten())); + codeInfoIndex = NonmovableArrays.createByteArray(TypeConversion.asU4(indexBuffer.getBytesWritten()), NmtCategory.Code); indexBuffer.toByteBuffer(NonmovableArrays.asByteBuffer(codeInfoIndex)); - codeInfoEncodings = NonmovableArrays.createByteArray(TypeConversion.asU4(encodingBuffer.getBytesWritten())); + codeInfoEncodings = NonmovableArrays.createByteArray(TypeConversion.asU4(encodingBuffer.getBytesWritten()), NmtCategory.Code); encodingBuffer.toByteBuffer(NonmovableArrays.asByteBuffer(codeInfoEncodings)); } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/DeoptimizationSourcePositionEncoder.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/DeoptimizationSourcePositionEncoder.java index 5c77dc245957..19908dd5523f 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/DeoptimizationSourcePositionEncoder.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/DeoptimizationSourcePositionEncoder.java @@ -29,17 +29,19 @@ import org.graalvm.collections.EconomicMap; import org.graalvm.collections.EconomicSet; import org.graalvm.collections.Equivalence; -import jdk.graal.compiler.core.common.util.FrequencyEncoder; -import jdk.graal.compiler.core.common.util.TypeConversion; -import jdk.graal.compiler.core.common.util.UnsafeArrayTypeWriter; -import jdk.graal.compiler.graph.NodeSourcePosition; import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.c.NonmovableArray; import com.oracle.svm.core.c.NonmovableArrays; import com.oracle.svm.core.c.NonmovableObjectArray; +import com.oracle.svm.core.nmt.NmtCategory; import com.oracle.svm.core.util.ByteArrayReader; +import jdk.graal.compiler.core.common.util.FrequencyEncoder; +import jdk.graal.compiler.core.common.util.TypeConversion; +import jdk.graal.compiler.core.common.util.UnsafeArrayTypeWriter; +import jdk.graal.compiler.graph.NodeSourcePosition; + public class DeoptimizationSourcePositionEncoder { private final FrequencyEncoder objectConstants; @@ -54,10 +56,10 @@ public void encodeAndInstall(List deoptSourcePositions, Code UnsafeArrayTypeWriter encodingBuffer = UnsafeArrayTypeWriter.create(ByteArrayReader.supportsUnalignedMemoryAccess()); EconomicMap sourcePositionStartOffsets = EconomicMap.create(Equivalence.IDENTITY_WITH_SYSTEM_HASHCODE); - NonmovableArray deoptStartOffsets = NonmovableArrays.createIntArray(deoptSourcePositions.size()); + NonmovableArray deoptStartOffsets = NonmovableArrays.createIntArray(deoptSourcePositions.size(), NmtCategory.Code); encodeSourcePositions(deoptSourcePositions, sourcePositionStartOffsets, deoptStartOffsets, encodingBuffer); - NonmovableArray deoptEncodings = NonmovableArrays.createByteArray(TypeConversion.asS4(encodingBuffer.getBytesWritten())); + NonmovableArray deoptEncodings = NonmovableArrays.createByteArray(TypeConversion.asS4(encodingBuffer.getBytesWritten()), NmtCategory.Code); encodingBuffer.toByteBuffer(NonmovableArrays.asByteBuffer(deoptEncodings)); install(target, deoptStartOffsets, deoptEncodings, encodedObjectConstants, deoptSourcePositions, adjuster); @@ -67,7 +69,7 @@ public void encodeAndInstall(List deoptSourcePositions, Code private static void install(CodeInfo target, NonmovableArray deoptStartOffsets, NonmovableArray deoptEncodings, Object[] encodedObjectConstants, List deoptSourcePositions, ReferenceAdjuster adjuster) { - NonmovableObjectArray deoptObjectConstants = adjuster.copyOfObjectArray(encodedObjectConstants); + NonmovableObjectArray deoptObjectConstants = adjuster.copyOfObjectArray(encodedObjectConstants, NmtCategory.Code); RuntimeCodeInfoAccess.setDeoptimizationMetadata(target, deoptStartOffsets, deoptEncodings, deoptObjectConstants); afterInstallation(deoptStartOffsets, deoptEncodings, deoptSourcePositions, deoptObjectConstants, adjuster); diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/FrameInfoEncoder.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/FrameInfoEncoder.java index 5e06b89ba6ca..1227003e734a 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/FrameInfoEncoder.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/FrameInfoEncoder.java @@ -56,6 +56,7 @@ import com.oracle.svm.core.meta.SharedField; import com.oracle.svm.core.meta.SharedMethod; import com.oracle.svm.core.meta.SharedType; +import com.oracle.svm.core.nmt.NmtCategory; import com.oracle.svm.core.sampler.CallStackFrameMethodData; import com.oracle.svm.core.sampler.CallStackFrameMethodInfo; import com.oracle.svm.core.util.ByteArrayReader; @@ -909,7 +910,7 @@ private NonmovableArray encodeFrameDatas() { assert frameMetadata.writeFrameVerificationInfo(data, encoders); } } - NonmovableArray frameInfoEncodings = NonmovableArrays.createByteArray(TypeConversion.asS4(encodingBuffer.getBytesWritten())); + NonmovableArray frameInfoEncodings = NonmovableArrays.createByteArray(TypeConversion.asS4(encodingBuffer.getBytesWritten()), NmtCategory.Code); encodingBuffer.toByteBuffer(NonmovableArrays.asByteBuffer(frameInfoEncodings)); return frameInfoEncodings; } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/ImageCodeInfo.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/ImageCodeInfo.java index fec3fde15470..4c8140249588 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/ImageCodeInfo.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/ImageCodeInfo.java @@ -41,6 +41,7 @@ import com.oracle.svm.core.c.NonmovableObjectArray; import com.oracle.svm.core.heap.UnknownObjectField; import com.oracle.svm.core.heap.UnknownPrimitiveField; +import com.oracle.svm.core.nmt.NmtCategory; import com.oracle.svm.core.util.VMError; import jdk.graal.compiler.word.Word; @@ -71,7 +72,7 @@ public class ImageCodeInfo { @Platforms(Platform.HOSTED_ONLY.class) ImageCodeInfo() { - NonmovableObjectArray objfields = NonmovableArrays.createObjectArray(Object[].class, CodeInfoImpl.OBJFIELDS_COUNT); + NonmovableObjectArray objfields = NonmovableArrays.createObjectArray(Object[].class, CodeInfoImpl.OBJFIELDS_COUNT, NmtCategory.Code); NonmovableArrays.setObject(objfields, CodeInfoImpl.NAME_OBJFIELD, CODE_INFO_NAME); // The image code info is never invalidated, so we consider it as always tethered. NonmovableArrays.setObject(objfields, CodeInfoImpl.TETHER_OBJFIELD, new CodeInfoTether(true)); diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/InstalledCodeObserverSupport.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/InstalledCodeObserverSupport.java index f2c7f21523a7..63a53f6691ca 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/InstalledCodeObserverSupport.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/InstalledCodeObserverSupport.java @@ -27,8 +27,6 @@ import java.util.ArrayList; import java.util.List; -import jdk.graal.compiler.code.CompilationResult; -import jdk.graal.compiler.debug.DebugContext; import org.graalvm.nativeimage.Platform; import org.graalvm.nativeimage.Platforms; import org.graalvm.word.Pointer; @@ -40,6 +38,10 @@ import com.oracle.svm.core.code.InstalledCodeObserver.InstalledCodeObserverHandle; import com.oracle.svm.core.feature.AutomaticallyRegisteredImageSingleton; import com.oracle.svm.core.meta.SharedMethod; +import com.oracle.svm.core.nmt.NmtCategory; + +import jdk.graal.compiler.code.CompilationResult; +import jdk.graal.compiler.debug.DebugContext; @AutomaticallyRegisteredImageSingleton public final class InstalledCodeObserverSupport { @@ -74,7 +76,7 @@ public static NonmovableArray installObservers(Inst if (observers.length == 0) { return NonmovableArrays.nullArray(); } - NonmovableArray observerHandles = NonmovableArrays.createWordArray(observers.length); + NonmovableArray observerHandles = NonmovableArrays.createWordArray(observers.length, NmtCategory.Code); for (int i = 0; i < observers.length; i++) { InstalledCodeObserverHandle handle = observers[i].install(); NonmovableArrays.setWord(observerHandles, i, handle); diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/InstantReferenceAdjuster.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/InstantReferenceAdjuster.java index bc577142320c..c7bafc4dfe7c 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/InstantReferenceAdjuster.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/InstantReferenceAdjuster.java @@ -31,6 +31,7 @@ import com.oracle.svm.core.c.NonmovableArrays; import com.oracle.svm.core.c.NonmovableObjectArray; import com.oracle.svm.core.meta.DirectSubstrateObjectConstant; +import com.oracle.svm.core.nmt.NmtCategory; import jdk.vm.ci.meta.JavaConstant; @@ -59,8 +60,8 @@ public void setConstantTargetAt(PointerBase address, int length, JavaConstant co @Override @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - public NonmovableObjectArray copyOfObjectArray(T[] source) { - return NonmovableArrays.copyOfObjectArray(source); + public NonmovableObjectArray copyOfObjectArray(T[] source, NmtCategory nmtCategory) { + return NonmovableArrays.copyOfObjectArray(source, nmtCategory); } @Override diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/ReferenceAdjuster.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/ReferenceAdjuster.java index 55c69b6fc7f3..124e31840391 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/ReferenceAdjuster.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/ReferenceAdjuster.java @@ -26,7 +26,6 @@ import java.nio.ByteOrder; -import jdk.graal.compiler.api.replacements.Fold; import org.graalvm.word.Pointer; import org.graalvm.word.PointerBase; @@ -35,7 +34,9 @@ import com.oracle.svm.core.c.NonmovableObjectArray; import com.oracle.svm.core.config.ConfigurationValues; import com.oracle.svm.core.heap.ReferenceAccess; +import com.oracle.svm.core.nmt.NmtCategory; +import jdk.graal.compiler.api.replacements.Fold; import jdk.vm.ci.meta.Constant; import jdk.vm.ci.meta.JavaConstant; @@ -53,8 +54,8 @@ public interface ReferenceAdjuster { void setConstantTargetAt(PointerBase address, int length, JavaConstant constant); @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - default NonmovableObjectArray copyOfObjectConstantArray(T[] constants) { - NonmovableObjectArray objects = NonmovableArrays.createObjectArray(Object[].class, constants.length); + default NonmovableObjectArray copyOfObjectConstantArray(T[] constants, NmtCategory nmtCategory) { + NonmovableObjectArray objects = NonmovableArrays.createObjectArray(Object[].class, constants.length, nmtCategory); for (int i = 0; i < constants.length; i++) { setConstantTargetInArray(objects, i, (JavaConstant) constants[i]); } @@ -62,7 +63,7 @@ default NonmovableObjectArray copyOfObjectConstantA } @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - NonmovableObjectArray copyOfObjectArray(T[] source); + NonmovableObjectArray copyOfObjectArray(T[] source, NmtCategory nmtCategory); /** Indicates whether all object references have been written. */ @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/RuntimeCodeCache.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/RuntimeCodeCache.java index 50a389d42d1e..6d2af1753d59 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/RuntimeCodeCache.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/RuntimeCodeCache.java @@ -46,6 +46,7 @@ import com.oracle.svm.core.deopt.Deoptimizer; import com.oracle.svm.core.deopt.SubstrateInstalledCode; import com.oracle.svm.core.heap.RestrictHeapAccess; +import com.oracle.svm.core.nmt.NmtCategory; import com.oracle.svm.core.option.RuntimeOptionKey; import com.oracle.svm.core.stack.JavaStackWalker; import com.oracle.svm.core.stack.StackFrameVisitor; @@ -192,7 +193,7 @@ private void enlargeTable() { if (newTableSize < INITIAL_TABLE_SIZE) { newTableSize = INITIAL_TABLE_SIZE; } - NonmovableArray newCodeInfos = NonmovableArrays.createWordArray(newTableSize); + NonmovableArray newCodeInfos = NonmovableArrays.createWordArray(newTableSize, NmtCategory.Code); if (codeInfos.isNonNull()) { NonmovableArrays.arraycopy(codeInfos, 0, newCodeInfos, 0, NonmovableArrays.lengthOf(codeInfos)); NonmovableArrays.releaseUnmanagedArray(codeInfos); diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/RuntimeCodeInfoAccess.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/RuntimeCodeInfoAccess.java index 334672f9fc6a..4b5d4b0a5d13 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/RuntimeCodeInfoAccess.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/RuntimeCodeInfoAccess.java @@ -26,10 +26,7 @@ import java.util.EnumSet; -import org.graalvm.nativeimage.ImageSingletons; -import org.graalvm.nativeimage.UnmanagedMemory; import org.graalvm.nativeimage.c.function.CodePointer; -import org.graalvm.nativeimage.impl.UnmanagedMemorySupport; import org.graalvm.word.Pointer; import org.graalvm.word.UnsignedWord; import org.graalvm.word.WordFactory; @@ -45,6 +42,9 @@ import com.oracle.svm.core.heap.CodeReferenceMapDecoder; import com.oracle.svm.core.heap.Heap; import com.oracle.svm.core.heap.ObjectReferenceVisitor; +import com.oracle.svm.core.memory.NativeMemory; +import com.oracle.svm.core.memory.NullableNativeMemory; +import com.oracle.svm.core.nmt.NmtCategory; import com.oracle.svm.core.os.CommittedMemoryProvider; import com.oracle.svm.core.util.DuplicatedInNativeCode; import com.oracle.svm.core.util.VMError; @@ -203,12 +203,12 @@ public static boolean walkObjectFields(CodeInfo info, ObjectReferenceVisitor vis } public static CodeInfo allocateMethodInfo() { - NonmovableObjectArray objectFields = NonmovableArrays.createObjectArray(Object[].class, CodeInfoImpl.OBJFIELDS_COUNT); + NonmovableObjectArray objectFields = NonmovableArrays.createObjectArray(Object[].class, CodeInfoImpl.OBJFIELDS_COUNT, NmtCategory.Code); return allocateMethodInfo(objectFields); } public static CodeInfo allocateMethodInfo(NonmovableObjectArray objectData) { - CodeInfoImpl info = UnmanagedMemory.calloc(CodeInfoAccess.getSizeOfCodeInfo()); + CodeInfoImpl info = NativeMemory.calloc(CodeInfoAccess.getSizeOfCodeInfo(), NmtCategory.Code); assert objectData.isNonNull() && NonmovableArrays.lengthOf(objectData) == CodeInfoImpl.OBJFIELDS_COUNT; info.setObjectFields(objectData); @@ -279,7 +279,7 @@ public static void free(CodeInfo info) { } impl.setState(CodeInfo.STATE_FREED); - ImageSingletons.lookup(UnmanagedMemorySupport.class).free(info); + NullableNativeMemory.free(info); } private static final NonmovableArrayAction GUARANTEE_ALL_OBJECTS_IN_IMAGE_HEAP_ACTION = new NonmovableArrayAction() { diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/RuntimeCodeInfoMemory.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/RuntimeCodeInfoMemory.java index 677a26df2e17..761481723d48 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/RuntimeCodeInfoMemory.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/RuntimeCodeInfoMemory.java @@ -42,6 +42,7 @@ import com.oracle.svm.core.deopt.SubstrateInstalledCode; import com.oracle.svm.core.heap.Heap; import com.oracle.svm.core.log.Log; +import com.oracle.svm.core.nmt.NmtCategory; import com.oracle.svm.core.thread.VMOperation; import com.oracle.svm.core.util.VMError; @@ -235,7 +236,7 @@ public boolean removeDuringGC(CodeInfo info) { private void add0(CodeInfo info) { addToSizeCounters(info); if (table.isNull()) { - table = NonmovableArrays.createWordArray(32); + table = NonmovableArrays.createWordArray(32, NmtCategory.Code); } int index; boolean resized; @@ -271,7 +272,7 @@ private boolean resize(int newLength) { return false; } NonmovableArray oldTable = table; - table = NonmovableArrays.createWordArray(newLength); + table = NonmovableArrays.createWordArray(newLength, NmtCategory.Code); for (int i = 0; i < oldLength; i++) { UntetheredCodeInfo tag = NonmovableArrays.getWord(oldTable, i); if (tag.isNonNull()) { diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/collections/AbstractUninterruptibleHashtable.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/collections/AbstractUninterruptibleHashtable.java index 8166fa91416a..f5b915281843 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/collections/AbstractUninterruptibleHashtable.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/collections/AbstractUninterruptibleHashtable.java @@ -24,16 +24,16 @@ */ package com.oracle.svm.core.collections; -import org.graalvm.nativeimage.ImageSingletons; import org.graalvm.nativeimage.Platform; import org.graalvm.nativeimage.Platforms; -import org.graalvm.nativeimage.impl.UnmanagedMemorySupport; import org.graalvm.word.Pointer; import org.graalvm.word.UnsignedWord; import org.graalvm.word.WordFactory; import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.UnmanagedMemoryUtil; +import com.oracle.svm.core.memory.NullableNativeMemory; +import com.oracle.svm.core.nmt.NmtCategory; /** * An uninterruptible hashtable with a fixed size that uses chaining in case of a collision. @@ -41,17 +41,19 @@ public abstract class AbstractUninterruptibleHashtable implements UninterruptibleHashtable { private static final int DEFAULT_TABLE_LENGTH = 2053; + private final NmtCategory nmtCategory; private final UninterruptibleEntry[] table; - protected int size; + private int size; @Platforms(Platform.HOSTED_ONLY.class) - public AbstractUninterruptibleHashtable() { - this(DEFAULT_TABLE_LENGTH); + public AbstractUninterruptibleHashtable(NmtCategory nmtCategory) { + this(nmtCategory, DEFAULT_TABLE_LENGTH); } @Platforms(Platform.HOSTED_ONLY.class) @SuppressWarnings("this-escape") - public AbstractUninterruptibleHashtable(int primeLength) { + public AbstractUninterruptibleHashtable(NmtCategory nmtCategory, int primeLength) { + this.nmtCategory = nmtCategory; this.table = createTable(primeLength); this.size = 0; } @@ -67,7 +69,7 @@ public AbstractUninterruptibleHashtable(int primeLength) { @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) protected UninterruptibleEntry copyToHeap(UninterruptibleEntry pointerOnStack, UnsignedWord sizeToAlloc) { - UninterruptibleEntry pointerOnHeap = ImageSingletons.lookup(UnmanagedMemorySupport.class).malloc(sizeToAlloc); + UninterruptibleEntry pointerOnHeap = NullableNativeMemory.malloc(sizeToAlloc, nmtCategory); if (pointerOnHeap.isNonNull()) { UnmanagedMemoryUtil.copy((Pointer) pointerOnStack, (Pointer) pointerOnHeap, sizeToAlloc); return pointerOnHeap; @@ -78,7 +80,7 @@ protected UninterruptibleEntry copyToHeap(UninterruptibleEntry pointerOnStack, U @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) protected void free(UninterruptibleEntry entry) { size--; - ImageSingletons.lookup(UnmanagedMemorySupport.class).free(entry); + NullableNativeMemory.free(entry); } @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) @@ -96,7 +98,7 @@ protected UninterruptibleEntry insertEntry(UninterruptibleEntry valueOnStack) { } @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - public void remove(UninterruptibleEntry valueOnStack) { + public boolean remove(UninterruptibleEntry valueOnStack) { int index = Integer.remainderUnsigned(valueOnStack.getHash(), DEFAULT_TABLE_LENGTH); UninterruptibleEntry entry = table[index]; UninterruptibleEntry prev = WordFactory.nullPointer(); @@ -108,11 +110,12 @@ public void remove(UninterruptibleEntry valueOnStack) { prev.setNext(entry.getNext()); } free(entry); - return; + return true; } prev = entry; entry = entry.getNext(); } + return false; } @Override diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/collections/GrowableWordArrayAccess.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/collections/GrowableWordArrayAccess.java index f9d7e0e3e337..6b3142d625bd 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/collections/GrowableWordArrayAccess.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/collections/GrowableWordArrayAccess.java @@ -24,16 +24,17 @@ */ package com.oracle.svm.core.collections; -import jdk.graal.compiler.api.replacements.Fold; -import jdk.graal.compiler.word.Word; -import org.graalvm.nativeimage.ImageSingletons; import org.graalvm.nativeimage.c.type.WordPointer; -import org.graalvm.nativeimage.impl.UnmanagedMemorySupport; import org.graalvm.word.Pointer; import org.graalvm.word.WordFactory; import com.oracle.svm.core.UnmanagedMemoryUtil; import com.oracle.svm.core.config.ConfigurationValues; +import com.oracle.svm.core.memory.NullableNativeMemory; +import com.oracle.svm.core.nmt.NmtCategory; + +import jdk.graal.compiler.api.replacements.Fold; +import jdk.graal.compiler.word.Word; public class GrowableWordArrayAccess { private static final int INITIAL_CAPACITY = 10; @@ -49,8 +50,8 @@ public static Word get(GrowableWordArray array, int i) { return array.getData().addressOf(i).read(); } - public static boolean add(GrowableWordArray array, Word element) { - if (array.getSize() == array.getCapacity() && !grow(array)) { + public static boolean add(GrowableWordArray array, Word element, NmtCategory nmtCategory) { + if (array.getSize() == array.getCapacity() && !grow(array, nmtCategory)) { return false; } @@ -61,14 +62,14 @@ public static boolean add(GrowableWordArray array, Word element) { public static void freeData(GrowableWordArray array) { if (array.isNonNull()) { - ImageSingletons.lookup(UnmanagedMemorySupport.class).free(array.getData()); + NullableNativeMemory.free(array.getData()); array.setData(WordFactory.nullPointer()); array.setSize(0); array.setCapacity(0); } } - private static boolean grow(GrowableWordArray array) { + private static boolean grow(GrowableWordArray array, NmtCategory nmtCategory) { int newCapacity = computeNewCapacity(array); if (newCapacity < 0) { /* Overflow. */ @@ -77,13 +78,13 @@ private static boolean grow(GrowableWordArray array) { assert newCapacity >= INITIAL_CAPACITY; WordPointer oldData = array.getData(); - WordPointer newData = ImageSingletons.lookup(UnmanagedMemorySupport.class).malloc(WordFactory.unsigned(newCapacity).multiply(wordSize())); + WordPointer newData = NullableNativeMemory.malloc(WordFactory.unsigned(newCapacity).multiply(wordSize()), nmtCategory); if (newData.isNull()) { return false; } UnmanagedMemoryUtil.copyForward((Pointer) oldData, (Pointer) newData, WordFactory.unsigned(array.getSize()).multiply(wordSize())); - ImageSingletons.lookup(UnmanagedMemorySupport.class).free(oldData); + NullableNativeMemory.free(oldData); array.setData(newData); array.setCapacity(newCapacity); diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/headers/LibC.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/headers/LibC.java index f23f371b99e1..65681458609d 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/headers/LibC.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/headers/LibC.java @@ -24,7 +24,6 @@ */ package com.oracle.svm.core.headers; -import jdk.graal.compiler.api.replacements.Fold; import org.graalvm.nativeimage.ImageSingletons; import org.graalvm.nativeimage.c.type.CCharPointer; import org.graalvm.nativeimage.c.type.CCharPointerPointer; @@ -34,6 +33,9 @@ import com.oracle.svm.core.Uninterruptible; +import jdk.graal.compiler.api.replacements.Fold; + +/** Platform-independent LibC support. */ public class LibC { public static final int EXIT_CODE_ABORT = 99; diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/headers/LibCSupport.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/headers/LibCSupport.java index 6965ee9b356d..b7ed0b46fdae 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/headers/LibCSupport.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/headers/LibCSupport.java @@ -31,7 +31,9 @@ import org.graalvm.word.UnsignedWord; import com.oracle.svm.core.Uninterruptible; +import com.oracle.svm.core.memory.NativeMemory; +/** Platform-independent LibC support. Don't use this class directly, use {@link LibC} instead. */ public interface LibCSupport { @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) int errno(); @@ -51,15 +53,19 @@ public interface LibCSupport { @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) T memset(T s, SignedWord c, UnsignedWord n); + /** Don't call this directly, see {@link NativeMemory} for more details. */ @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) T malloc(UnsignedWord size); + /** Don't call this directly, see {@link NativeMemory} for more details. */ @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) T calloc(UnsignedWord nmemb, UnsignedWord size); + /** Don't call this directly, see {@link NativeMemory} for more details. */ @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) T realloc(PointerBase ptr, UnsignedWord size); + /** Don't call this directly, see {@link NativeMemory} for more details. */ @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) void free(PointerBase ptr); diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ReferenceMapEncoder.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ReferenceMapEncoder.java index e1f987790055..edaf84a07a39 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ReferenceMapEncoder.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ReferenceMapEncoder.java @@ -33,13 +33,14 @@ import java.util.PrimitiveIterator; import java.util.Set; -import jdk.graal.compiler.core.common.util.TypeConversion; -import jdk.graal.compiler.core.common.util.UnsafeArrayTypeWriter; - import com.oracle.svm.core.c.NonmovableArray; import com.oracle.svm.core.c.NonmovableArrays; +import com.oracle.svm.core.nmt.NmtCategory; import com.oracle.svm.core.util.ByteArrayReader; +import jdk.graal.compiler.core.common.util.TypeConversion; +import jdk.graal.compiler.core.common.util.UnsafeArrayTypeWriter; + public abstract class ReferenceMapEncoder { public interface OffsetIterator extends PrimitiveIterator.OfInt { @Override @@ -99,7 +100,7 @@ public NonmovableArray encodeAll() { encodeAll(sortedEntries); int length = TypeConversion.asS4(writeBuffer.getBytesWritten()); - NonmovableArray array = NonmovableArrays.createByteArray(length); + NonmovableArray array = NonmovableArrays.createByteArray(length, NmtCategory.Code); writeBuffer.toByteBuffer(NonmovableArrays.asByteBuffer(array)); return array; } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/dump/HeapDumpMetadata.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/dump/HeapDumpMetadata.java index 04ef357f2423..942812612c05 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/dump/HeapDumpMetadata.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/dump/HeapDumpMetadata.java @@ -24,7 +24,6 @@ */ package com.oracle.svm.core.heap.dump; -import jdk.graal.compiler.api.replacements.Fold; import org.graalvm.nativeimage.ImageSingletons; import org.graalvm.nativeimage.Platform; import org.graalvm.nativeimage.Platforms; @@ -34,7 +33,6 @@ import org.graalvm.nativeimage.c.struct.RawStructure; import org.graalvm.nativeimage.c.struct.SizeOf; import org.graalvm.nativeimage.c.type.CCharPointer; -import org.graalvm.nativeimage.impl.UnmanagedMemorySupport; import org.graalvm.word.Pointer; import org.graalvm.word.PointerBase; import org.graalvm.word.UnsignedWord; @@ -47,11 +45,15 @@ import com.oracle.svm.core.heap.ObjectVisitor; import com.oracle.svm.core.heap.UnknownObjectField; import com.oracle.svm.core.hub.DynamicHub; +import com.oracle.svm.core.memory.NullableNativeMemory; +import com.oracle.svm.core.nmt.NmtCategory; import com.oracle.svm.core.util.coder.ByteStream; import com.oracle.svm.core.util.coder.ByteStreamAccess; import com.oracle.svm.core.util.coder.NativeCoder; import com.oracle.svm.core.util.coder.Pack200Coder; +import jdk.graal.compiler.api.replacements.Fold; + /** * Provides access to the encoded heap dump metadata that was prepared at image build-time. * @@ -140,19 +142,19 @@ public boolean initialize() { * structures so that we can abort right away in case that an allocation fails. */ UnsignedWord classInfosSize = WordFactory.unsigned(classInfoCount).multiply(SizeOf.get(ClassInfo.class)); - classInfos = ImageSingletons.lookup(UnmanagedMemorySupport.class).calloc(classInfosSize); + classInfos = NullableNativeMemory.calloc(classInfosSize, NmtCategory.HeapDump); if (classInfos.isNull()) { return false; } UnsignedWord fieldStartsSize = WordFactory.unsigned(totalFieldCount).multiply(SizeOf.get(FieldInfoPointer.class)); - fieldInfoTable = ImageSingletons.lookup(UnmanagedMemorySupport.class).calloc(fieldStartsSize); + fieldInfoTable = NullableNativeMemory.calloc(fieldStartsSize, NmtCategory.HeapDump); if (fieldInfoTable.isNull()) { return false; } UnsignedWord fieldNameTableSize = WordFactory.unsigned(fieldNameCount).multiply(SizeOf.get(FieldNamePointer.class)); - fieldNameTable = ImageSingletons.lookup(UnmanagedMemorySupport.class).calloc(fieldNameTableSize); + fieldNameTable = NullableNativeMemory.calloc(fieldNameTableSize, NmtCategory.HeapDump); if (fieldNameTable.isNull()) { return false; } @@ -214,13 +216,13 @@ public boolean initialize() { * Must always be called, regardless if {@link #initialize} returned true or false. */ public void teardown() { - ImageSingletons.lookup(UnmanagedMemorySupport.class).free(classInfos); + NullableNativeMemory.free(classInfos); classInfos = WordFactory.nullPointer(); - ImageSingletons.lookup(UnmanagedMemorySupport.class).free(fieldInfoTable); + NullableNativeMemory.free(fieldInfoTable); fieldInfoTable = WordFactory.nullPointer(); - ImageSingletons.lookup(UnmanagedMemorySupport.class).free(fieldNameTable); + NullableNativeMemory.free(fieldNameTable); fieldNameTable = WordFactory.nullPointer(); } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/dump/HeapDumpSupportImpl.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/dump/HeapDumpSupportImpl.java index 95cffbb587bd..4f6db2db3570 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/dump/HeapDumpSupportImpl.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/dump/HeapDumpSupportImpl.java @@ -28,8 +28,6 @@ import java.io.IOException; -import jdk.graal.compiler.api.replacements.Fold; -import org.graalvm.nativeimage.ImageSingletons; import org.graalvm.nativeimage.Platform; import org.graalvm.nativeimage.Platforms; import org.graalvm.nativeimage.StackValue; @@ -37,7 +35,6 @@ import org.graalvm.nativeimage.c.struct.RawStructure; import org.graalvm.nativeimage.c.struct.SizeOf; import org.graalvm.nativeimage.c.type.CCharPointer; -import org.graalvm.nativeimage.impl.UnmanagedMemorySupport; import org.graalvm.word.Pointer; import org.graalvm.word.WordFactory; @@ -47,6 +44,7 @@ import com.oracle.svm.core.heap.RestrictHeapAccess; import com.oracle.svm.core.heap.VMOperationInfos; import com.oracle.svm.core.log.Log; +import com.oracle.svm.core.memory.UntrackedNullableNativeMemory; import com.oracle.svm.core.os.RawFileOperationSupport; import com.oracle.svm.core.os.RawFileOperationSupport.FileCreationMode; import com.oracle.svm.core.os.RawFileOperationSupport.RawFileDescriptor; @@ -55,6 +53,8 @@ import com.oracle.svm.core.thread.VMOperation; import com.oracle.svm.core.util.TimeUtils; +import jdk.graal.compiler.api.replacements.Fold; + public class HeapDumpSupportImpl extends HeapDumping { private final HeapDumpWriter writer; private final HeapDumpOperation heapDumpOperation; @@ -77,7 +77,7 @@ public void initializeDumpHeapOnOutOfMemoryError() { @Override public void teardownDumpHeapOnOutOfMemoryError() { - ImageSingletons.lookup(UnmanagedMemorySupport.class).free(outOfMemoryHeapDumpPath); + UntrackedNullableNativeMemory.free(outOfMemoryHeapDumpPath); outOfMemoryHeapDumpPath = WordFactory.nullPointer(); } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/dump/HeapDumpWriter.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/dump/HeapDumpWriter.java index 63338d5f2ee2..30fb293d6c67 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/dump/HeapDumpWriter.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/dump/HeapDumpWriter.java @@ -71,6 +71,7 @@ import com.oracle.svm.core.hub.DynamicHub; import com.oracle.svm.core.hub.LayoutEncoding; import com.oracle.svm.core.log.Log; +import com.oracle.svm.core.nmt.NmtCategory; import com.oracle.svm.core.os.BufferedFileOperationSupport; import com.oracle.svm.core.os.BufferedFileOperationSupport.BufferedFile; import com.oracle.svm.core.os.RawFileOperationSupport.RawFileDescriptor; @@ -443,7 +444,7 @@ private boolean dumpHeap0(RawFileDescriptor fd) { private boolean initialize(RawFileDescriptor fd) { assert topLevelRecordBegin == -1 && subRecordBegin == -1 && !error; - this.f = file().allocate(fd); + this.f = file().allocate(fd, NmtCategory.HeapDump); if (f.isNull()) { return false; } @@ -1340,7 +1341,7 @@ public void initialize(GrowableWordArray largeObjects) { @RestrictHeapAccess(access = NO_ALLOCATION, reason = "Heap dumping must not allocate.") public boolean visitObject(Object obj) { if (isLarge(obj)) { - boolean added = GrowableWordArrayAccess.add(largeObjects, Word.objectToUntrackedPointer(obj)); + boolean added = GrowableWordArrayAccess.add(largeObjects, Word.objectToUntrackedPointer(obj), NmtCategory.HeapDump); if (!added) { Log.log().string("Failed to add an element to the large object list. Heap dump will be incomplete.").newline(); } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/SunMiscSubstitutions.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/SunMiscSubstitutions.java index ed8e8891c086..0d8a683fcc28 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/SunMiscSubstitutions.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/SunMiscSubstitutions.java @@ -29,9 +29,7 @@ import java.lang.reflect.Method; import java.security.ProtectionDomain; -import jdk.graal.compiler.nodes.extended.MembarNode; import org.graalvm.nativeimage.ImageSingletons; -import org.graalvm.nativeimage.UnmanagedMemory; import org.graalvm.nativeimage.impl.UnsafeMemorySupport; import org.graalvm.word.WordFactory; @@ -46,27 +44,30 @@ import com.oracle.svm.core.hub.DynamicHub; import com.oracle.svm.core.hub.LayoutEncoding; import com.oracle.svm.core.hub.PredefinedClassesSupport; -import com.oracle.svm.core.nmt.NmtFlag; +import com.oracle.svm.core.memory.NativeMemory; +import com.oracle.svm.core.nmt.NmtCategory; import com.oracle.svm.core.os.VirtualMemoryProvider; import com.oracle.svm.core.util.VMError; +import jdk.graal.compiler.nodes.extended.MembarNode; + @TargetClass(className = "jdk.internal.misc.Unsafe") @SuppressWarnings({"static-method", "unused"}) final class Target_jdk_internal_misc_Unsafe_Core { @Substitute private long allocateMemory0(long bytes) { - return UnmanagedMemory.malloc(WordFactory.unsigned(bytes), NmtFlag.mtOther.ordinal()).rawValue(); + return NativeMemory.malloc(WordFactory.unsigned(bytes), NmtCategory.Unsafe).rawValue(); } @Substitute private long reallocateMemory0(long address, long bytes) { - return UnmanagedMemory.realloc(WordFactory.unsigned(address), WordFactory.unsigned(bytes), NmtFlag.mtOther.ordinal()).rawValue(); + return NativeMemory.realloc(WordFactory.unsigned(address), WordFactory.unsigned(bytes), NmtCategory.Unsafe).rawValue(); } @Substitute private void freeMemory0(long address) { - UnmanagedMemory.free(WordFactory.unsigned(address)); + NativeMemory.free(WordFactory.unsigned(address)); } @Substitute diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/TimeZoneSubstitutions.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/TimeZoneSubstitutions.java index 6f3fb6cf8741..925f43e24b0f 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/TimeZoneSubstitutions.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/TimeZoneSubstitutions.java @@ -32,11 +32,8 @@ import java.util.TimeZone; import org.graalvm.collections.EconomicMap; -import jdk.graal.compiler.options.Option; -import jdk.graal.compiler.options.OptionKey; import org.graalvm.nativeimage.ImageSingletons; import org.graalvm.nativeimage.Platforms; -import org.graalvm.nativeimage.UnmanagedMemory; import org.graalvm.nativeimage.c.type.CCharPointer; import org.graalvm.nativeimage.c.type.CTypeConversion; import org.graalvm.nativeimage.impl.InternalPlatform; @@ -51,9 +48,13 @@ import com.oracle.svm.core.feature.AutomaticallyRegisteredFeature; import com.oracle.svm.core.feature.InternalFeature; import com.oracle.svm.core.handles.PrimitiveArrayView; +import com.oracle.svm.core.memory.UntrackedNullableNativeMemory; import com.oracle.svm.core.option.HostedOptionKey; import com.oracle.svm.core.util.VMError; +import jdk.graal.compiler.options.Option; +import jdk.graal.compiler.options.OptionKey; + /** * The following classes aim to provide full support for time zones for native-image. This * substitution is necessary due to the reliance on JAVA_HOME in the JDK. @@ -100,7 +101,7 @@ private static String getSystemTimeZoneID(String javaHome) { CCharPointer tzId = LibCHelper.SVM_FindJavaTZmd(tzMappingsPtr, contentLen); String result = CTypeConversion.toJavaString(tzId); // SVM_FindJavaTZmd returns a newly allocated string - UnmanagedMemory.untrackedFree(tzId); + UntrackedNullableNativeMemory.free(tzId); return result; } finally { if (refContent != null) { diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jfr/JfrBufferAccess.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jfr/JfrBufferAccess.java index f83bd41dd1d0..8c3a4389bbf8 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jfr/JfrBufferAccess.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jfr/JfrBufferAccess.java @@ -24,20 +24,20 @@ */ package com.oracle.svm.core.jfr; -import jdk.graal.compiler.api.replacements.Fold; -import org.graalvm.nativeimage.ImageSingletons; import org.graalvm.nativeimage.c.struct.SizeOf; -import org.graalvm.nativeimage.impl.UnmanagedMemorySupport; import org.graalvm.word.Pointer; import org.graalvm.word.UnsignedWord; import org.graalvm.word.WordFactory; import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.config.ConfigurationValues; -import com.oracle.svm.core.nmt.NmtFlag; +import com.oracle.svm.core.memory.NullableNativeMemory; +import com.oracle.svm.core.nmt.NmtCategory; import com.oracle.svm.core.thread.VMOperation; import com.oracle.svm.core.util.UnsignedUtils; +import jdk.graal.compiler.api.replacements.Fold; + /** * Used to access the raw memory of a {@link JfrBuffer}. */ @@ -62,7 +62,7 @@ public static JfrBuffer allocate(JfrBufferType bufferType) { @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static JfrBuffer allocate(UnsignedWord dataSize, JfrBufferType bufferType) { UnsignedWord headerSize = JfrBufferAccess.getHeaderSize(); - JfrBuffer result = ImageSingletons.lookup(UnmanagedMemorySupport.class).malloc(headerSize.add(dataSize), NmtFlag.mtTracing.ordinal()); + JfrBuffer result = NullableNativeMemory.malloc(headerSize.add(dataSize), NmtCategory.JFR); if (result.isNonNull()) { result.setSize(dataSize); result.setBufferType(bufferType); @@ -75,7 +75,7 @@ public static JfrBuffer allocate(UnsignedWord dataSize, JfrBufferType bufferType @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static void free(JfrBuffer buffer) { - ImageSingletons.lookup(UnmanagedMemorySupport.class).free(buffer); + NullableNativeMemory.free(buffer); } @Uninterruptible(reason = "Prevent safepoints as those could change the flushed position.") diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jfr/JfrBufferList.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jfr/JfrBufferList.java index 17c202d7693a..320a0922a66c 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jfr/JfrBufferList.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jfr/JfrBufferList.java @@ -26,10 +26,8 @@ package com.oracle.svm.core.jfr; -import org.graalvm.nativeimage.ImageSingletons; import org.graalvm.nativeimage.Platform; import org.graalvm.nativeimage.Platforms; -import org.graalvm.nativeimage.impl.UnmanagedMemorySupport; import org.graalvm.word.WordFactory; import com.oracle.svm.core.Uninterruptible; @@ -75,7 +73,7 @@ public void teardown() { } JfrBufferNode next = node.getNext(); - ImageSingletons.lookup(UnmanagedMemorySupport.class).free(node); + JfrBufferNodeAccess.free(node); node = next; } head = WordFactory.nullPointer(); diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jfr/JfrBufferNodeAccess.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jfr/JfrBufferNodeAccess.java index 9102ec67abbb..a72e5118208c 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jfr/JfrBufferNodeAccess.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jfr/JfrBufferNodeAccess.java @@ -27,15 +27,14 @@ package com.oracle.svm.core.jfr; import org.graalvm.nativeimage.CurrentIsolate; -import org.graalvm.nativeimage.ImageSingletons; import org.graalvm.nativeimage.c.struct.SizeOf; import org.graalvm.nativeimage.c.type.CIntPointer; -import org.graalvm.nativeimage.impl.UnmanagedMemorySupport; import org.graalvm.word.Pointer; import org.graalvm.word.WordFactory; -import com.oracle.svm.core.nmt.NmtFlag; import com.oracle.svm.core.Uninterruptible; +import com.oracle.svm.core.memory.NullableNativeMemory; +import com.oracle.svm.core.nmt.NmtCategory; import com.oracle.svm.core.thread.NativeSpinLockUtils; import com.oracle.svm.core.thread.VMOperation; @@ -48,7 +47,7 @@ private JfrBufferNodeAccess() { @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static JfrBufferNode allocate(JfrBuffer buffer) { - JfrBufferNode node = ImageSingletons.lookup(UnmanagedMemorySupport.class).malloc(SizeOf.unsigned(JfrBufferNode.class), NmtFlag.mtTracing.ordinal()); + JfrBufferNode node = NullableNativeMemory.malloc(SizeOf.unsigned(JfrBufferNode.class), NmtCategory.JFR); if (node.isNonNull()) { node.setBuffer(buffer); node.setNext(WordFactory.nullPointer()); @@ -60,7 +59,7 @@ public static JfrBufferNode allocate(JfrBuffer buffer) { @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static void free(JfrBufferNode node) { - ImageSingletons.lookup(UnmanagedMemorySupport.class).free(node); + NullableNativeMemory.free(node); } /** Should be used instead of {@link JfrBufferNode#getBuffer}. */ diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jfr/JfrStackTraceRepository.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jfr/JfrStackTraceRepository.java index ee885c2a0c0c..c8b98185ca65 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jfr/JfrStackTraceRepository.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jfr/JfrStackTraceRepository.java @@ -33,7 +33,6 @@ import org.graalvm.nativeimage.c.struct.RawStructure; import org.graalvm.nativeimage.c.struct.SizeOf; import org.graalvm.nativeimage.c.type.CIntPointer; -import org.graalvm.nativeimage.impl.UnmanagedMemorySupport; import org.graalvm.word.Pointer; import org.graalvm.word.UnsignedWord; import org.graalvm.word.WordFactory; @@ -51,7 +50,8 @@ import com.oracle.svm.core.jfr.traceid.JfrTraceIdEpoch; import com.oracle.svm.core.jfr.utils.JfrVisited; import com.oracle.svm.core.locks.VMMutex; -import com.oracle.svm.core.nmt.NmtFlag; +import com.oracle.svm.core.memory.NullableNativeMemory; +import com.oracle.svm.core.nmt.NmtCategory; import com.oracle.svm.core.sampler.SamplerSampleWriter; import com.oracle.svm.core.sampler.SamplerSampleWriterData; import com.oracle.svm.core.sampler.SamplerSampleWriterDataAccess; @@ -194,7 +194,7 @@ private JfrStackTraceTableEntry getOrPutStackTrace0(Pointer start, UnsignedWord * the thread-local buffer to the C heap because the thread-local buffer will be * overwritten or freed at some point. */ - Pointer to = ImageSingletons.lookup(UnmanagedMemorySupport.class).malloc(size, NmtFlag.mtTracing.ordinal()); + Pointer to = NullableNativeMemory.malloc(size, NmtCategory.JFR); if (to.isNonNull()) { UnmanagedMemoryUtil.copy(start, to, size); entry.setRawStackTrace(to); @@ -206,7 +206,8 @@ private JfrStackTraceTableEntry getOrPutStackTrace0(Pointer start, UnsignedWord } /* Hashtable entry allocation failed. */ - ImageSingletons.lookup(UnmanagedMemorySupport.class).free(to); + NullableNativeMemory.free(to); + to = WordFactory.nullPointer(); } /* Some allocation failed. */ @@ -304,6 +305,11 @@ public interface JfrStackTraceTableEntry extends JfrVisited { public static final class JfrStackTraceTable extends AbstractUninterruptibleHashtable { private static long nextId; + @Platforms(Platform.HOSTED_ONLY.class) + JfrStackTraceTable() { + super(NmtCategory.JFR); + } + @Override @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) protected JfrStackTraceTableEntry[] createTable(int size) { @@ -334,7 +340,7 @@ protected UninterruptibleEntry copyToHeap(UninterruptibleEntry valueOnStack) { protected void free(UninterruptibleEntry entry) { JfrStackTraceTableEntry stackTraceEntry = (JfrStackTraceTableEntry) entry; /* The base method will free only the entry itself, not the pointer with stacktrace. */ - ImageSingletons.lookup(UnmanagedMemorySupport.class).free(stackTraceEntry.getRawStackTrace()); + NullableNativeMemory.free(stackTraceEntry.getRawStackTrace()); super.free(entry); } } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jfr/JfrSymbolRepository.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jfr/JfrSymbolRepository.java index 8949a12c50cb..a5b6f6bee8e9 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jfr/JfrSymbolRepository.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jfr/JfrSymbolRepository.java @@ -24,8 +24,6 @@ */ package com.oracle.svm.core.jfr; -import jdk.graal.compiler.core.common.SuppressFBWarnings; -import jdk.graal.compiler.word.Word; import org.graalvm.nativeimage.Platform; import org.graalvm.nativeimage.Platforms; import org.graalvm.nativeimage.StackValue; @@ -43,6 +41,10 @@ import com.oracle.svm.core.jdk.UninterruptibleUtils.CharReplacer; import com.oracle.svm.core.jfr.traceid.JfrTraceIdEpoch; import com.oracle.svm.core.locks.VMMutex; +import com.oracle.svm.core.nmt.NmtCategory; + +import jdk.graal.compiler.core.common.SuppressFBWarnings; +import jdk.graal.compiler.word.Word; /** * In Native Image, we use {@link java.lang.String} objects that live in the image heap as symbols. @@ -179,6 +181,11 @@ private interface JfrSymbol extends UninterruptibleEntry { private static class JfrSymbolHashtable extends AbstractUninterruptibleHashtable { private static long nextId; + @Platforms(Platform.HOSTED_ONLY.class) + JfrSymbolHashtable() { + super(NmtCategory.JFR); + } + @Override @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) protected JfrSymbol[] createTable(int size) { diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jfr/utils/JfrVisitedTable.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jfr/utils/JfrVisitedTable.java index 0aa3e41ff71c..593c54d2d56f 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jfr/utils/JfrVisitedTable.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jfr/utils/JfrVisitedTable.java @@ -25,13 +25,20 @@ package com.oracle.svm.core.jfr.utils; +import org.graalvm.nativeimage.Platform; +import org.graalvm.nativeimage.Platforms; import org.graalvm.nativeimage.c.struct.SizeOf; import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.collections.AbstractUninterruptibleHashtable; import com.oracle.svm.core.collections.UninterruptibleEntry; +import com.oracle.svm.core.nmt.NmtCategory; public final class JfrVisitedTable extends AbstractUninterruptibleHashtable { + @Platforms(Platform.HOSTED_ONLY.class) + public JfrVisitedTable() { + super(NmtCategory.JFR); + } @Override @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jni/JNIJavaVMList.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jni/JNIJavaVMList.java index 24131c95471d..8966b02ff48c 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jni/JNIJavaVMList.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jni/JNIJavaVMList.java @@ -29,8 +29,6 @@ import static org.graalvm.word.WordFactory.unsigned; import static org.graalvm.word.WordFactory.zero; -import jdk.graal.compiler.word.Word; -import org.graalvm.nativeimage.UnmanagedMemory; import org.graalvm.nativeimage.c.struct.SizeOf; import org.graalvm.nativeimage.c.type.CIntPointer; import org.graalvm.nativeimage.c.type.WordPointer; @@ -42,6 +40,10 @@ import com.oracle.svm.core.c.CGlobalDataFactory; import com.oracle.svm.core.jni.headers.JNIJavaVM; import com.oracle.svm.core.jni.headers.JNIJavaVMPointer; +import com.oracle.svm.core.memory.NativeMemory; +import com.oracle.svm.core.nmt.NmtCategory; + +import jdk.graal.compiler.word.Word; /** * A process-global, lock-free list of JavaVM pointers. Implemented as arrays in native memory which @@ -76,7 +78,7 @@ public static void addJavaVM(JNIJavaVM newEntry) { Pointer p = nextPointer.readWord(0); if (p.isNull()) { // No empty slots, create new array UnsignedWord newCapacity = capacity.notEqual(0) ? capacity.multiply(2) : INITIAL_CAPACITY; - Pointer newArray = UnmanagedMemory.calloc(newCapacity.add(2 /* capacity and next */).multiply(wordSize)); + Pointer newArray = NativeMemory.calloc(newCapacity.add(2 /* capacity and next */).multiply(wordSize), NmtCategory.JNI); newArray.writeWord(0, newCapacity); newArray.writeWord(wordSize, newEntry); p = nextPointer.compareAndSwapWord(0, nullPointer(), newArray, ANY_LOCATION); @@ -84,7 +86,7 @@ public static void addJavaVM(JNIJavaVM newEntry) { return; } // Another thread already created and linked a new array, continue in that array - UnmanagedMemory.free(newArray); + NativeMemory.free(newArray); } capacity = p.readWord(0); p = p.add(wordSize); diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jni/functions/JNIInvocationInterface.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jni/functions/JNIInvocationInterface.java index 60d6d3b04e1e..730742e77080 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jni/functions/JNIInvocationInterface.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jni/functions/JNIInvocationInterface.java @@ -24,8 +24,6 @@ */ package com.oracle.svm.core.jni.functions; -import jdk.graal.compiler.serviceprovider.IsolateUtil; -import org.graalvm.nativeimage.ImageSingletons; import org.graalvm.nativeimage.Isolate; import org.graalvm.nativeimage.LogHandler; import org.graalvm.nativeimage.StackValue; @@ -36,7 +34,6 @@ import org.graalvm.nativeimage.c.type.CCharPointerPointer; import org.graalvm.nativeimage.c.type.CIntPointer; import org.graalvm.nativeimage.c.type.WordPointer; -import org.graalvm.nativeimage.impl.UnmanagedMemorySupport; import org.graalvm.word.Pointer; import org.graalvm.word.UnsignedWord; import org.graalvm.word.WordFactory; @@ -73,6 +70,7 @@ import com.oracle.svm.core.jni.headers.JNIJavaVMPointer; import com.oracle.svm.core.jni.headers.JNIVersion; import com.oracle.svm.core.log.FunctionPointerLogHandler; +import com.oracle.svm.core.memory.UntrackedNullableNativeMemory; import com.oracle.svm.core.monitor.MonitorInflationCause; import com.oracle.svm.core.monitor.MonitorSupport; import com.oracle.svm.core.snippets.ImplicitExceptions; @@ -80,6 +78,8 @@ import com.oracle.svm.core.thread.PlatformThreads; import com.oracle.svm.core.util.Utf8; +import jdk.graal.compiler.serviceprovider.IsolateUtil; + /** * Implementation of the JNI invocation API for interacting with a Java VM without having an * existing context and without linking against the Java VM library beforehand. @@ -128,7 +128,7 @@ static int enter(JNIJavaVMPointer vmBuf, JNIEnvironmentPointer penv, JNIJavaVMIn int vmArgc = vmArgs.getNOptions(); if (vmArgc > 0) { UnsignedWord size = SizeOf.unsigned(CCharPointerPointer.class).multiply(vmArgc + 1); - CCharPointerPointer argv = ImageSingletons.lookup(UnmanagedMemorySupport.class).malloc(size); + CCharPointerPointer argv = UntrackedNullableNativeMemory.malloc(size); if (argv.isNull()) { return JNIErrors.JNI_ENOMEM(); } @@ -169,7 +169,7 @@ static int enter(JNIJavaVMPointer vmBuf, JNIEnvironmentPointer penv, JNIJavaVMIn int code = CEntryPointActions.enterCreateIsolate(params); if (params.isNonNull()) { - ImageSingletons.lookup(UnmanagedMemorySupport.class).free(params.getArgv()); + UntrackedNullableNativeMemory.free(params.getArgv()); params = WordFactory.nullPointer(); } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jvmstat/CHeapPerfMemoryProvider.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jvmstat/CHeapPerfMemoryProvider.java index d7ba95251947..ac1417b0e57b 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jvmstat/CHeapPerfMemoryProvider.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jvmstat/CHeapPerfMemoryProvider.java @@ -26,11 +26,12 @@ import java.nio.ByteBuffer; -import org.graalvm.nativeimage.UnmanagedMemory; import org.graalvm.word.Pointer; import org.graalvm.word.WordFactory; import com.oracle.svm.core.jdk.DirectByteBufferUtil; +import com.oracle.svm.core.memory.NativeMemory; +import com.oracle.svm.core.nmt.NmtCategory; /** * Allocates a buffer with a minimal size that only contains the performance data header (see @@ -42,14 +43,14 @@ public class CHeapPerfMemoryProvider implements PerfMemoryProvider { @Override public ByteBuffer create() { int size = PerfMemoryPrologue.getPrologueSize(); - memory = UnmanagedMemory.calloc(size); + memory = NativeMemory.calloc(size, NmtCategory.JvmStat); return DirectByteBufferUtil.allocate(memory.rawValue(), size); } @Override public void teardown() { if (memory.isNonNull()) { - UnmanagedMemory.free(memory); + NativeMemory.free(memory); memory = WordFactory.nullPointer(); } } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jvmstat/PerfMemory.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jvmstat/PerfMemory.java index 223f2481a1a3..75b4dfb96d96 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jvmstat/PerfMemory.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jvmstat/PerfMemory.java @@ -29,12 +29,10 @@ import java.nio.ByteBuffer; import java.nio.ByteOrder; -import jdk.graal.compiler.word.Word; import org.graalvm.nativeimage.CurrentIsolate; import org.graalvm.nativeimage.ImageSingletons; import org.graalvm.nativeimage.Platform; import org.graalvm.nativeimage.Platforms; -import org.graalvm.nativeimage.UnmanagedMemory; import org.graalvm.word.LocationIdentity; import org.graalvm.word.Pointer; import org.graalvm.word.WordFactory; @@ -45,6 +43,10 @@ import com.oracle.svm.core.c.CGlobalDataFactory; import com.oracle.svm.core.jdk.DirectByteBufferUtil; import com.oracle.svm.core.jdk.Target_java_nio_Buffer; +import com.oracle.svm.core.memory.NativeMemory; +import com.oracle.svm.core.nmt.NmtCategory; + +import jdk.graal.compiler.word.Word; /** * Provides access to the underlying OS-specific memory that stores the performance data. @@ -148,7 +150,7 @@ public synchronized Word allocate(int size) { if (used + size >= capacity) { PerfMemoryPrologue.addOverflow(rawMemory, size); // Always return a valid pointer (external tools won't see this memory though). - Word result = UnmanagedMemory.calloc(size); + Word result = NativeMemory.calloc(size, NmtCategory.JvmStat); addOverflowMemory(result); return result; } @@ -198,7 +200,7 @@ public void teardown() { if (overflowMemory != null) { for (int i = 0; i < overflowMemory.length; i++) { - UnmanagedMemory.free(overflowMemory[i]); + NativeMemory.free(overflowMemory[i]); } overflowMemory = null; } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/memory/NativeMemory.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/memory/NativeMemory.java new file mode 100644 index 000000000000..31929d623cd1 --- /dev/null +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/memory/NativeMemory.java @@ -0,0 +1,148 @@ +/* + * Copyright (c) 2024, 2024, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2023, 2023, Red Hat Inc. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package com.oracle.svm.core.memory; + +import static com.oracle.svm.core.Uninterruptible.CALLED_FROM_UNINTERRUPTIBLE_CODE; + +import org.graalvm.nativeimage.UnmanagedMemory; +import org.graalvm.nativeimage.impl.UnmanagedMemorySupport; +import org.graalvm.word.PointerBase; +import org.graalvm.word.UnsignedWord; +import org.graalvm.word.WordFactory; + +import com.oracle.svm.core.Uninterruptible; +import com.oracle.svm.core.nmt.NmtCategory; + +/** + * Internal API for managing native memory. This class supports native memory tracking (NMT) and is + * therefore preferred over the public API class {@link UnmanagedMemory} and its + * {@link UnmanagedMemorySupport implementations}. + * + * All methods that allocate native memory throw an {@link OutOfMemoryError} if the memory + * allocation fails. If native memory needs to be allocated from uninterruptible code, use + * {@link NullableNativeMemory} instead. + * + * Note that NMT may cause segfaults in certain scenarios: + *
    + *
  • Native memory that was allocated outside of Java (e.g., in a C library) or via some API that + * does not support NMT (e.g., {@link UnmanagedMemory}) may only be freed with + * {@link UntrackedNullableNativeMemory#free}. This is necessary because NMT assumes that each + * allocation has a custom header, which isn't true for such memory.
    • + *
  • NMT accesses the image heap. If native memory needs to be allocated before the image heap is + * mapped or after it was unmapped, {@link UntrackedNullableNativeMemory} must be used.
    • + *
+ */ +public class NativeMemory { + /** + * Allocates {@code size} bytes of native memory. The content of the memory is undefined. + *

+ * This method never returns a null pointer but instead throws an {@link OutOfMemoryError} when + * allocation fails. + */ + public static T malloc(UnsignedWord size, NmtCategory category) { + T result = NullableNativeMemory.malloc(size, category); + if (result.isNull()) { + throw new OutOfMemoryError("Memory allocation failed: malloc returned null."); + } + return result; + } + + /** + * Allocates {@code size} bytes of native memory. The content of the memory is undefined. + *

+ * This method never returns a null pointer but instead throws an {@link OutOfMemoryError} when + * allocation fails. + */ + public static T malloc(int size, NmtCategory category) { + assert size >= 0; + return malloc(WordFactory.unsigned(size), category); + } + + /** + * Allocates {@code size} bytes of native memory. The content of the memory is set to 0. + *

+ * This method never returns a null pointer, but instead throws an {@link OutOfMemoryError} when + * allocation fails. + */ + public static T calloc(UnsignedWord size, NmtCategory category) { + T result = NullableNativeMemory.calloc(size, category); + if (result.isNull()) { + throw new OutOfMemoryError("Memory allocation failed: calloc returned null."); + } + return result; + } + + /** + * Allocates {@code size} bytes of native memory. The content of the memory is set to 0. + *

+ * This method never returns a null pointer, but instead throws an {@link OutOfMemoryError} when + * allocation fails. + */ + public static T calloc(int size, NmtCategory category) { + assert size >= 0; + return calloc(WordFactory.unsigned(size), category); + } + + /** + * Changes the size of the provided native memory to {@code size} bytes. If the new size is + * larger than the old size, the content of the additional memory is undefined. + *

+ * This method never returns a null pointer, but instead throws an {@link OutOfMemoryError} when + * allocation fails. In that case, the old data is not deallocated and remains unchanged. + */ + public static T realloc(T ptr, UnsignedWord size, NmtCategory category) { + T result = NullableNativeMemory.realloc(ptr, size, category); + if (result.isNull()) { + throw new OutOfMemoryError("Memory allocation failed: realloc returned null."); + } + return result; + } + + /** + * Changes the size of the provided native memory to {@code size} bytes. If the new size is + * larger than the old size, the content of the additional memory is undefined. + *

+ * This method never returns a null pointer, but instead throws an {@link OutOfMemoryError} when + * allocation fails. In that case, the old data is not deallocated and remains unchanged. + */ + public static T realloc(T ptr, int size, NmtCategory category) { + assert size >= 0; + return realloc(ptr, WordFactory.unsigned(size), category); + } + + /** + * Frees native memory that was previously allocated using methods of this class. This method is + * a no-op if the given pointer is {@code null}. + *

+ * Note that this method must NOT be used to free memory that was allocated via other + * classes (e.g., {@link UnmanagedMemorySupport}) or outside of Native Image code (e.g., in a C + * library). + */ + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + public static void free(PointerBase ptr) { + NullableNativeMemory.free(ptr); + } +} diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/memory/NullableNativeMemory.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/memory/NullableNativeMemory.java new file mode 100644 index 000000000000..864a2d933e7f --- /dev/null +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/memory/NullableNativeMemory.java @@ -0,0 +1,184 @@ +/* + * Copyright (c) 2024, 2024, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2023, 2023, Red Hat Inc. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package com.oracle.svm.core.memory; + +import static com.oracle.svm.core.Uninterruptible.CALLED_FROM_UNINTERRUPTIBLE_CODE; + +import org.graalvm.nativeimage.UnmanagedMemory; +import org.graalvm.nativeimage.impl.UnmanagedMemorySupport; +import org.graalvm.word.PointerBase; +import org.graalvm.word.UnsignedWord; +import org.graalvm.word.WordFactory; + +import com.oracle.svm.core.Uninterruptible; +import com.oracle.svm.core.VMInspectionOptions; +import com.oracle.svm.core.nmt.NativeMemoryTracking; +import com.oracle.svm.core.nmt.NmtCategory; +import com.oracle.svm.core.nmt.NmtMallocHeader; + +/** + * Internal API for managing native memory. This class supports native memory tracking (NMT) and is + * therefore preferred over the public API class {@link UnmanagedMemory} and its + * {@link UnmanagedMemorySupport implementations}. + * + * All methods that allocate native memory return {@code null} if the memory allocation fails. + */ +public class NullableNativeMemory { + /** + * Allocates {@code size} bytes of native memory. The content of the memory is undefined. + *

+ * This method returns a null pointer when allocation fails. + */ + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + public static T malloc(UnsignedWord size, NmtCategory flag) { + T outerPointer = UntrackedNullableNativeMemory.malloc(getAllocationSize(size)); + return track(outerPointer, size, flag); + } + + /** + * Allocates {@code size} bytes of native memory. The content of the memory is undefined. + *

+ * This method returns a null pointer when allocation fails. + */ + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + public static T malloc(int size, NmtCategory flag) { + assert size >= 0; + return malloc(WordFactory.unsigned(size), flag); + } + + /** + * Allocates {@code size} bytes of native memory. The content of the memory is set to 0. + *

+ * This method returns a null pointer when allocation fails. + */ + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + public static T calloc(UnsignedWord size, NmtCategory flag) { + T outerPointer = UntrackedNullableNativeMemory.calloc(getAllocationSize(size)); + return track(outerPointer, size, flag); + } + + /** + * Allocates {@code size} bytes of native memory. The content of the memory is set to 0. + *

+ * This method returns a null pointer when allocation fails. + */ + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + public static T calloc(int size, NmtCategory flag) { + assert size >= 0; + return calloc(WordFactory.unsigned(size), flag); + } + + /** + * Changes the size of the provided native memory to {@code size} bytes. If the new size is + * larger than the old size, the content of the additional memory is undefined. + *

+ * This method returns a null pointer when allocation fails. In that case, the old data is not + * deallocated and remains unchanged. + */ + @SuppressWarnings("unchecked") + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + public static T realloc(T ptr, UnsignedWord size, NmtCategory flag) { + if (ptr.isNull()) { + return malloc(size, flag); + } else if (!VMInspectionOptions.hasNativeMemoryTrackingSupport()) { + return UntrackedNullableNativeMemory.realloc(ptr, getAllocationSize(size)); + } + + /* Query the NMT information for the old allocation. */ + NmtMallocHeader header = NativeMemoryTracking.getHeader(ptr); + T oldOuterPointer = (T) header; + int oldCategory = header.getCategory(); + UnsignedWord oldSize = header.getAllocationSize(); + + /* Try to realloc. */ + T newOuterPointer = UntrackedNullableNativeMemory.realloc(oldOuterPointer, getAllocationSize(size)); + if (newOuterPointer.isNull()) { + return WordFactory.nullPointer(); + } + + oldOuterPointer = WordFactory.nullPointer(); + + /* Only untrack the old block if the allocation was successful. */ + NativeMemoryTracking.singleton().untrack(oldSize, oldCategory); + return track(newOuterPointer, size, flag); + } + + /** + * Changes the size of the provided native memory to {@code size} bytes. If the new size is + * larger than the old size, the content of the additional memory is undefined. + *

+ * This method returns a null pointer when allocation fails. In that case, the old data is not + * deallocated and remains unchanged. + */ + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + public static T realloc(T ptr, int size, NmtCategory flag) { + assert size >= 0; + return realloc(ptr, WordFactory.unsigned(size), flag); + } + + /** + * Frees native memory that was previously allocated using methods of this class. This method is + * a no-op if the given pointer is {@code null}. + *

+ * Note that this method must NOT be used to free memory that was allocated via other + * classes (e.g., {@link UnmanagedMemorySupport}) or outside of Native Image code (e.g., in a C + * library). + */ + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + public static void free(PointerBase ptr) { + if (ptr.isNull()) { + return; + } + + PointerBase outerPtr = untrack(ptr); + UntrackedNullableNativeMemory.free(outerPtr); + } + + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + private static UnsignedWord getAllocationSize(UnsignedWord size) { + if (VMInspectionOptions.hasNativeMemoryTrackingSupport()) { + return size.add(NativeMemoryTracking.sizeOfNmtHeader()); + } + return size; + } + + @SuppressWarnings("unchecked") + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + private static T track(T outerPtr, UnsignedWord size, NmtCategory flag) { + if (VMInspectionOptions.hasNativeMemoryTrackingSupport() && outerPtr.isNonNull()) { + return (T) NativeMemoryTracking.singleton().track(outerPtr, size, flag); + } + return outerPtr; + } + + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + private static PointerBase untrack(PointerBase innerPtr) { + if (VMInspectionOptions.hasNativeMemoryTrackingSupport()) { + return NativeMemoryTracking.singleton().untrack(innerPtr); + } + return innerPtr; + } +} diff --git a/substratevm/src/com.oracle.svm.core.windows/src/com/oracle/svm/core/windows/WindowsUnmanagedMemorySupportImpl.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/memory/UnmanagedMemorySupportImpl.java similarity index 68% rename from substratevm/src/com.oracle.svm.core.windows/src/com/oracle/svm/core/windows/WindowsUnmanagedMemorySupportImpl.java rename to substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/memory/UnmanagedMemorySupportImpl.java index 3f9d85915fea..9c41d7ce1b7c 100644 --- a/substratevm/src/com.oracle.svm.core.windows/src/com/oracle/svm/core/windows/WindowsUnmanagedMemorySupportImpl.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/memory/UnmanagedMemorySupportImpl.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018, 2018, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2018, 2024, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -22,9 +22,7 @@ * or visit www.oracle.com if you need additional information or have any * questions. */ -package com.oracle.svm.core.windows; - -import jdk.graal.compiler.api.replacements.Fold; +package com.oracle.svm.core.memory; import org.graalvm.nativeimage.ImageSingletons; import org.graalvm.nativeimage.impl.UnmanagedMemorySupport; @@ -33,20 +31,27 @@ import org.graalvm.word.WordFactory; import com.oracle.svm.core.Uninterruptible; -import com.oracle.svm.core.feature.AutomaticallyRegisteredImageSingleton; +import com.oracle.svm.core.feature.AutomaticallyRegisteredFeature; +import com.oracle.svm.core.feature.InternalFeature; import com.oracle.svm.core.headers.LibCSupport; +import com.oracle.svm.core.util.UnsignedUtils; + +import jdk.graal.compiler.api.replacements.Fold; -@AutomaticallyRegisteredImageSingleton(UnmanagedMemorySupport.class) -class WindowsUnmanagedMemorySupportImpl implements UnmanagedMemorySupport { +/** + * Delegates to the libc-specific memory management functions. Some platforms use a different + * implementation. + */ +class UnmanagedMemorySupportImpl implements UnmanagedMemorySupport { @Override @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public T malloc(UnsignedWord size) { - return libc().malloc(size); - } - - @Override - public T malloc(UnsignedWord size, int flag) { - return libc().malloc(size); + /* + * Some libc implementations may return a nullptr when the size is 0. We use a minimum size + * of 1 to ensure that we always get a unique pointer if the allocation succeeds. + */ + UnsignedWord allocationSize = UnsignedUtils.max(size, WordFactory.unsigned(1)); + return libc().malloc(allocationSize); } @Override @@ -55,35 +60,30 @@ public T calloc(UnsignedWord size) { return libc().calloc(WordFactory.unsigned(1), size); } - @Override - public T calloc(UnsignedWord size, int flag) { - return libc().calloc(WordFactory.unsigned(1), size); - } - @Override @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public T realloc(T ptr, UnsignedWord size) { return libc().realloc(ptr, size); } - @Override - public T realloc(T ptr, UnsignedWord size, int flag) { - return libc().realloc(ptr, size); - } - @Override @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void free(PointerBase ptr) { libc().free(ptr); } - @Override - public void untrackedFree(PointerBase ptr) { - libc().free(ptr); - } - @Fold static LibCSupport libc() { return ImageSingletons.lookup(LibCSupport.class); } } + +@AutomaticallyRegisteredFeature +class UnmanagedMemorySupportFeature implements InternalFeature { + @Override + public void duringSetup(DuringSetupAccess access) { + if (!ImageSingletons.contains(UnmanagedMemorySupport.class)) { + ImageSingletons.add(UnmanagedMemorySupport.class, new UnmanagedMemorySupportImpl()); + } + } +} diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/memory/UntrackedNullableNativeMemory.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/memory/UntrackedNullableNativeMemory.java new file mode 100644 index 000000000000..d05fa8041444 --- /dev/null +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/memory/UntrackedNullableNativeMemory.java @@ -0,0 +1,135 @@ +/* + * Copyright (c) 2024, 2024, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package com.oracle.svm.core.memory; + +import static com.oracle.svm.core.Uninterruptible.CALLED_FROM_UNINTERRUPTIBLE_CODE; + +import org.graalvm.nativeimage.ImageSingletons; +import org.graalvm.nativeimage.impl.UnmanagedMemorySupport; +import org.graalvm.word.PointerBase; +import org.graalvm.word.UnsignedWord; +import org.graalvm.word.WordFactory; + +import com.oracle.svm.core.Uninterruptible; + +import jdk.graal.compiler.api.replacements.Fold; + +/** + * Manages native memory. This class explicitly does NOT support native memory tracking + * (NMT). It can therefore be used for the following use cases: + *

    + *
  • Allocate native memory that is later freed outside of Native Image code (e.g., in a C + * library).
    • + *
  • Free native memory that was allocated outside of Native Image code (e.g., in a C + * library).
    • + *
+ */ +public class UntrackedNullableNativeMemory { + /** + * Allocates {@code size} bytes of native memory. The content of the memory is undefined. + *

+ * This method returns a null pointer when allocation fails. + */ + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + public static T malloc(UnsignedWord size) { + return memory().malloc(size); + } + + /** + * Allocates {@code size} bytes of native memory. The content of the memory is undefined. + *

+ * This method returns a null pointer when allocation fails. + */ + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + public static T malloc(int size) { + assert size >= 0; + return malloc(WordFactory.unsigned(size)); + } + + /** + * Allocates {@code size} bytes of native memory. The content of the memory is set to 0. + *

+ * This method returns a null pointer when allocation fails. + */ + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + public static T calloc(UnsignedWord size) { + return memory().calloc(size); + } + + /** + * Allocates {@code size} bytes of native memory. The content of the memory is set to 0. + *

+ * This method returns a null pointer when allocation fails. + */ + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + public static T calloc(int size) { + assert size >= 0; + return calloc(WordFactory.unsigned(size)); + } + + /** + * Changes the size of the provided native memory to {@code size} bytes. If the new size is + * larger than the old size, the content of the additional memory is undefined. + *

+ * This method returns a null pointer when allocation fails. In that case, the old data is not + * deallocated and remains unchanged. + */ + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + public static T realloc(T ptr, UnsignedWord size) { + return memory().realloc(ptr, size); + } + + /** + * Changes the size of the provided native memory to {@code size} bytes. If the new size is + * larger than the old size, the content of the additional memory is undefined. + *

+ * This method returns a null pointer when allocation fails. In that case, the old data is not + * deallocated and remains unchanged. + */ + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + public static T realloc(T ptr, int size) { + assert size >= 0; + return realloc(ptr, WordFactory.unsigned(size)); + } + + /** + * Frees native memory that was previously allocated using methods of this class or that was + * allocated outside of Java (e.g., in a C library). This method is a no-op if the given pointer + * is {@code null}. + *

+ * Note that this method must NOT be used to free memory that was allocated via classes + * that support native memory tracking (e.g., {@link NativeMemory} or + * {@link NullableNativeMemory}). + */ + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + public static void free(PointerBase ptr) { + memory().free(ptr); + } + + @Fold + static UnmanagedMemorySupport memory() { + return ImageSingletons.lookup(UnmanagedMemorySupport.class); + } +} diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/HasNmtSupport.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/HasNmtSupport.java deleted file mode 100644 index 19607542b0a0..000000000000 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/HasNmtSupport.java +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Copyright (c) 2023, 2023, Oracle and/or its affiliates. All rights reserved. - * Copyright (c) 2023, 2023, Red Hat Inc. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. Oracle designates this - * particular file as subject to the "Classpath" exception as provided - * by Oracle in the LICENSE file that accompanied this code. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - */ - -package com.oracle.svm.core.nmt; - -import java.util.function.BooleanSupplier; -import jdk.graal.compiler.api.replacements.Fold; -import org.graalvm.nativeimage.ImageSingletons; - -public class HasNmtSupport implements BooleanSupplier { - @Override - public boolean getAsBoolean() { - return get(); - } - - @Fold - public static boolean get() { - return ImageSingletons.contains(NativeMemoryTracking.class); - } -} diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/NativeMemoryTracking.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/NativeMemoryTracking.java index 984731c961be..491735517687 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/NativeMemoryTracking.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/NativeMemoryTracking.java @@ -26,472 +26,141 @@ package com.oracle.svm.core.nmt; -import com.oracle.svm.core.SubstrateOptions; -import com.oracle.svm.core.Uninterruptible; -import com.oracle.svm.core.config.ConfigurationValues; -import com.oracle.svm.core.headers.LibCSupport; -import com.oracle.svm.core.jdk.RuntimeSupport; -import com.oracle.svm.core.thread.JavaSpinLockUtils; -import com.oracle.svm.core.util.UnsignedUtils; -import com.oracle.svm.util.LogUtils; - -import jdk.graal.compiler.api.replacements.Fold; -import jdk.internal.misc.Unsafe; +import static com.oracle.svm.core.Uninterruptible.CALLED_FROM_UNINTERRUPTIBLE_CODE; import org.graalvm.nativeimage.ImageSingletons; import org.graalvm.nativeimage.Platform; import org.graalvm.nativeimage.Platforms; import org.graalvm.nativeimage.c.struct.SizeOf; -import org.graalvm.nativeimage.impl.UnmanagedMemorySupport; import org.graalvm.word.Pointer; import org.graalvm.word.PointerBase; import org.graalvm.word.UnsignedWord; import org.graalvm.word.WordFactory; +import com.oracle.svm.core.SubstrateOptions; +import com.oracle.svm.core.Uninterruptible; +import com.oracle.svm.core.jdk.RuntimeSupport; +import com.oracle.svm.core.memory.NativeMemory; +import com.oracle.svm.core.util.UnsignedUtils; + +import jdk.graal.compiler.api.replacements.Fold; + /** - * This is the class that handles native memory tracking (NMT). There are two components to NMT: - * tracking malloc/realloc/calloc, and tracking virtual memory usage. - *

- *

- *

- * Malloc/realloc/calloc: Malloc headers ({@link MallocHeader}) are used for caching data about the - * allocation. This is necessary because that data is needed to deaccount the tracked memory when it - * is freed. There are some scenarios where an implementation dependent on malloc header requires - * special precautions: - *

- * - *
    - *
  1. When native memory is allocated outside of java. This might be in a C library for example. - * When it comes time to free the memory, it is important that the NMT infrastructure, if enabled, - * does not assume a malloc header exists and attempt to access it. - * {@link UnmanagedMemorySupport#untrackedFree(PointerBase)} must be used in this case. - *
  2. When raw {@link LibCSupport} allocation methods are called directly. This is problematic - * because such allocated blocks will not contain malloc headers. If NMT is enabled it will assume - * headers exist and access incorrect memory locations. This must be avoided. - *
  3. During the phase pre-NMT initialization. Before runtime options are parsed, it is unknown - * whether NMT should be enabled/disabled. During this time, it is assumed that NMT will later be - * enabled, so malloc headers are allocated. This increases the accuracy of the tracking. However, - * if NMT is later disabled upon initialization, there will be a mix of allocated blocks that - * have headers and do some that do not . This is a problem when it comes time to free the - * blocks. For this reason, there are methods in this class that handle pre-init allocations using a - * look-up table to cache addresses seen before initialization. This is the same reason NMT cannot - * be enabled/disabled after initialization. - *
  4. If native allocations are done before the image heap is mapped. This is not strictly a - * problem related to malloc headers, but must still be avoided. If such situations are unavoidable, - * NMT code must be bypassed using raw {@link LibCSupport} methods for both allocation and - * deallocation. - *
+ * This class implements native memory tracking (NMT). There are two components to NMT: tracking + * memory allocations (malloc/realloc/calloc), and tracking virtual memory usage (not supported + * yet). + * + * For tracking memory allocations, we have an internal API (see {@link NativeMemory}) that adds a + * custom {@link NmtMallocHeader header} to each allocation if NMT is enabled. This header stores + * data that is needed to properly untrack the memory when it is freed. */ public class NativeMemoryTracking { - private static final Unsafe U = Unsafe.getUnsafe(); - private static final long LOCK_OFFSET = U.objectFieldOffset(NativeMemoryTracking.class, "preInitLock"); - - /** - * Can't use VmMutex because it tracks owners and this class may be used by unattached threads - * (calloc). This lock is only used during the pre-init phase. - */ - @SuppressWarnings("unused") private volatile int preInitLock; - - private final PreInitTable preInitTable; - private volatile boolean enabled; - private volatile boolean initialized; - - private MallocMemorySnapshot mallocMemorySnapshot; + private static final UnsignedWord ALIGNMENT = WordFactory.unsigned(16); + private static final int MAGIC = 0xF0F1F2F3; + private final NmtMallocMemorySnapshot mallocMemorySnapshot = new NmtMallocMemorySnapshot(); @Platforms(Platform.HOSTED_ONLY.class) public NativeMemoryTracking() { - enabled = true; - initialized = false; - mallocMemorySnapshot = new MallocMemorySnapshot(); - preInitTable = new PreInitTable(); - } - - @Fold - static UnsignedWord getHeaderSize0() { - return UnsignedUtils.roundUp(SizeOf.unsigned(MallocHeader.class), WordFactory.unsigned(ConfigurationValues.getTarget().wordSize)); } @Fold - static LibCSupport libc() { - return ImageSingletons.lookup(LibCSupport.class); - } - - @Fold - static NativeMemoryTracking get() { + public static NativeMemoryTracking singleton() { return ImageSingletons.lookup(NativeMemoryTracking.class); } - /** This must be called after the image heap is mapped. */ - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - public static void initialize(boolean enabled) { - if (HasNmtSupport.get()) { - get().initialize0(enabled); - } - } - - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - private void initialize0(boolean enabled) { - assert !initialized; - // Block until all in-progress pre-init operations are completed - lockNoTransition(); - try { - this.enabled = enabled; - // After this point, the preInit table becomes read only. - initialized = true; - } finally { - unlock(); - } - } - - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - public static UnsignedWord getHeaderSize() { - if (HasNmtSupport.get() && get().enabled) { - return getHeaderSize0(); - } - return WordFactory.zero(); - } - - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - public static long getAllocationSize(Pointer outerPointer) { - if (HasNmtSupport.get() && get().enabled) { - MallocHeader header = (MallocHeader) outerPointer; - return header.getSize(); - } - return -1; - } - - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - public static int getAllocationCategory(Pointer outerPointer) { - if (HasNmtSupport.get() && get().enabled) { - MallocHeader header = (MallocHeader) outerPointer; - return header.getFlag(); - } - return -1; - } - - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - public static Pointer recordMalloc(Pointer outerPointer, UnsignedWord size, int flag) { - if (HasNmtSupport.get() && outerPointer.isNonNull()) { - return get().recordMalloc0(outerPointer, size, flag); - } - return outerPointer; - } - - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - private Pointer recordMalloc0(Pointer outerPointer, UnsignedWord size, int flag) { - if (enabled) { - UnsignedWord headerSize = getHeaderSize0(); - mallocMemorySnapshot.getInfoByCategory(flag).recordMalloc(size); - mallocMemorySnapshot.getInfoByCategory(NmtFlag.mtNMT.ordinal()).recordMalloc(headerSize); - mallocMemorySnapshot.getTotalInfo().recordMalloc(size.add(headerSize)); - return initializeHeader(outerPointer, size, flag); - } - return outerPointer; - } - - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - private static Pointer initializeHeader(Pointer outerPointer, UnsignedWord size, int flag) { - MallocHeader mallocHeader = (MallocHeader) outerPointer; - mallocHeader.setSize(size.rawValue()); - mallocHeader.setFlag(flag); - return ((Pointer) mallocHeader).add(NativeMemoryTracking.getHeaderSize0()); - } - - /** - * This is only needed for {@link PreInitTable} since we must use raw malloc/free there but - * still want to track its allocations. - */ - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - public static void recordMallocWithoutHeader(UnsignedWord size, int flag) { - if (HasNmtSupport.get()) { - get().recordMallocWithoutHeader0(size, flag); - } - } - - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - private void recordMallocWithoutHeader0(UnsignedWord size, int flag) { - if (enabled) { - mallocMemorySnapshot.getInfoByCategory(flag).recordMalloc(size); - mallocMemorySnapshot.getTotalInfo().recordMalloc(size); - } - } - - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - public static void deaccountMalloc(PointerBase innerPtr) { - if (HasNmtSupport.get() && get().enabled) { - MallocHeader header = (MallocHeader) ((Pointer) innerPtr).subtract(getHeaderSize0()); - get().deaccountMalloc0(header.getSize(), header.getFlag()); - } - } - - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - public static void deaccountMalloc(long size, int flag) { - if (HasNmtSupport.get() && size > 0 && flag >= 0) { - get().deaccountMalloc0(size, flag); - } - } - - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - private void deaccountMalloc0(long size, int flag) { - if (enabled) { - mallocMemorySnapshot.getInfoByCategory(flag).deaccountMalloc(size); - mallocMemorySnapshot.getInfoByCategory(NmtFlag.mtNMT.ordinal()).deaccountMalloc(getHeaderSize0().rawValue()); - mallocMemorySnapshot.getTotalInfo().deaccountMalloc(size + getHeaderSize0().rawValue()); - } - } - - /** - * This is only needed for {@link PreInitTable} since we must use raw malloc/free there but - * still want to track its allocations. - */ - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - public static void deaccountMallocWithoutHeader(long size, int flag) { - if (HasNmtSupport.get() && size > 0 && flag >= 0) { - get().deaccountMallocWithoutHeader0(size, flag); - } + @Fold + public static UnsignedWord sizeOfNmtHeader() { + /* + * Align the allocation payload to 16 bytes (assuming that the platform-specific malloc + * implementation returns a pointer that is aligned to >= 16 bytes). + */ + return UnsignedUtils.roundUp(SizeOf.unsigned(NmtMallocHeader.class), ALIGNMENT); } - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - private void deaccountMallocWithoutHeader0(long size, int flag) { - if (enabled) { - mallocMemorySnapshot.getInfoByCategory(flag).deaccountMalloc(size); - mallocMemorySnapshot.getTotalInfo().deaccountMalloc(size); - } - } + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + public Pointer track(PointerBase outerPtr, UnsignedWord size, NmtCategory category) { + /* Initialize the header. */ + NmtMallocHeader mallocHeader = (NmtMallocHeader) outerPtr; + mallocHeader.setAllocationSize(size); + mallocHeader.setCategory(category.ordinal()); + assert setMagic(mallocHeader); - public static long getMallocByCategory(NmtFlag flag) { - if (HasNmtSupport.get()) { - return get().mallocMemorySnapshot.getInfoByCategory(flag.ordinal()).getSize(); - } - return -1; + /* Track the memory. */ + Pointer innerPtr = ((Pointer) outerPtr).add(sizeOfNmtHeader()); + track(innerPtr); + return innerPtr; } - /** Prints stats contained in the current snapshot. */ - public static void printStats() { - if (HasNmtSupport.get()) { - get().printStats0(); - } + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + private static boolean setMagic(NmtMallocHeader mallocHeader) { + mallocHeader.setMagic(MAGIC); + return true; } - private void printStats0() { - if (!enabled) { + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + public void track(PointerBase innerPtr) { + if (innerPtr.isNull()) { return; } - LogUtils.info("Total current malloc size:" + mallocMemorySnapshot.getTotalInfo().getSize() + "B"); - LogUtils.info("Total current malloc count:" + mallocMemorySnapshot.getTotalInfo().getCount()); - for (int i = 0; i < NmtFlag.values().length; i++) { - LogUtils.info(NmtFlag.values()[i].getName() + " current malloc size:" + mallocMemorySnapshot.getInfoByCategory(i).getSize() + "B"); - LogUtils.info(NmtFlag.values()[i].getName() + " current malloc count:" + mallocMemorySnapshot.getInfoByCategory(i).getCount()); - } + NmtMallocHeader header = getHeader(innerPtr); + UnsignedWord nmtHeaderSize = sizeOfNmtHeader(); + UnsignedWord allocationSize = header.getAllocationSize(); + UnsignedWord totalSize = allocationSize.add(nmtHeaderSize); + mallocMemorySnapshot.getInfoByCategory(header.getCategory()).track(allocationSize); + mallocMemorySnapshot.getInfoByCategory(NmtCategory.NMT).track(nmtHeaderSize); + mallocMemorySnapshot.getTotalInfo().track(totalSize); } - /** - * This method is needed because locking is required to make the allocation, LUT insertion, and - * initialization check atomic. If the malloc was handled here, returnAddress holds the location - * of the malloc block payload. - */ - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - public static boolean handlePreInitMallocs(UnsignedWord size, int flag, ReturnAddress returnAddress) { - if (HasNmtSupport.get()) { - return get().handlePreInitMallocs0(size, flag, returnAddress); + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + public PointerBase untrack(PointerBase innerPtr) { + if (innerPtr.isNull()) { + return WordFactory.nullPointer(); } - return false; - } - /** - * Special handling is needed here becuase we need ot check initialization state and add an - * address to the LUT. - */ - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - private boolean handlePreInitMallocs0(UnsignedWord size, int flag, ReturnAddress returnAddress) { - // For speed, check if initialized before acquiring lock - if (!initialized) { - lockNoTransition(); - try { - // Double check after acquiring lock. If it's since been initialized, proceed - // normally. - if (initialized) { - return false; - } - // Still uninitialized. Allocate and add to LUT. - Pointer outerPointer = libc().malloc(size.add(getHeaderSize0())); - recordPreInitAlloc(outerPointer, size, flag, returnAddress); - return true; - } finally { - unlock(); - } - } - return false; + NmtMallocHeader header = getHeader(innerPtr); + untrack(header.getAllocationSize(), header.getCategory()); + return header; } - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - public static boolean handlePreInitCallocs(UnsignedWord size, int flag, ReturnAddress returnAddress) { - if (HasNmtSupport.get()) { - return get().handlePreInitCallocs0(size, flag, returnAddress); - } - return false; + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + public void untrack(UnsignedWord size, int category) { + mallocMemorySnapshot.getInfoByCategory(category).untrack(size); + mallocMemorySnapshot.getInfoByCategory(NmtCategory.NMT).untrack(sizeOfNmtHeader()); + mallocMemorySnapshot.getTotalInfo().untrack(size.add(sizeOfNmtHeader())); } - /** - * Similar to - * {@link NativeMemoryTracking#handlePreInitMallocs0(UnsignedWord, int, ReturnAddress)}. - */ - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - private boolean handlePreInitCallocs0(UnsignedWord size, int flag, ReturnAddress returnAddress) { - if (!initialized) { - lockNoTransition(); - try { - if (initialized) { - return false; - } - Pointer outerPointer = libc().calloc(WordFactory.unsigned(1), size.add(getHeaderSize())); - recordPreInitAlloc(outerPointer, size, flag, returnAddress); - return true; - } finally { - unlock(); - } - } - return false; + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + public static NmtMallocHeader getHeader(PointerBase innerPtr) { + NmtMallocHeader result = (NmtMallocHeader) ((Pointer) innerPtr).subtract(sizeOfNmtHeader()); + assert result.getMagic() == MAGIC : "bad NMT malloc header"; + return result; } - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - public static boolean handlePreInitReallocs(Pointer oldInnerPtr, UnsignedWord size, int flag, ReturnAddress returnAddress) { - if (HasNmtSupport.get()) { - return get().handlePreInitReallocs0(oldInnerPtr, size, flag, returnAddress); - } - return false; + public long getUsedMemory(NmtCategory category) { + return mallocMemorySnapshot.getInfoByCategory(category).getUsed(); } - /** - * This method is needed because locking is required to make the allocation, LUT insertion, and - * initialization check atomic. - */ - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - private boolean handlePreInitReallocs0(Pointer oldInnerPtr, UnsignedWord size, int flag, ReturnAddress returnAddress) { - // For speed, check if initialized before acquiring lock - if (!initialized) { - lockNoTransition(); - try { - // Double check after acquiring lock. If still uninitialized, perform realloc and do - // LUT updates. - if (!initialized) { - // Retrieve necessary data from the old block - Pointer oldOuterPointer = oldInnerPtr.subtract(NativeMemoryTracking.getHeaderSize0()); - long oldSize = getAllocationSize(oldOuterPointer); - int oldCategory = getAllocationCategory(oldOuterPointer); - - // Perform the realloc - Pointer newOuterPointer = libc().realloc(oldOuterPointer, size.add(NativeMemoryTracking.getHeaderSize0())); - - if (newOuterPointer.isNonNull()) { - // Deaccount old block. Order matters here in case old/new have same - // address. - deaccountMalloc0(oldSize, oldCategory); - preInitTable.remove(oldInnerPtr); - - // Account new block - recordPreInitAlloc(newOuterPointer, size, flag, returnAddress); - } else { - returnAddress.set(WordFactory.nullPointer()); - } - return true; - } - } finally { - unlock(); - } - } - - // Post init. - if (!enabled && preInitTable.get(oldInnerPtr).isNonNull()) { - // There is a header from pre-init time, but tracking has since been disabled. - - // Malloc a new block with the given size and no header - Pointer newBlock = libc().malloc(size); - returnAddress.set(newBlock); - if (newBlock.isNonNull()) { - // Copy payload from original block to new block - Pointer oldOuterPointer = oldInnerPtr.subtract(getHeaderSize0()); - UnsignedWord oldSize = WordFactory.unsigned(getAllocationSize(oldOuterPointer)); - UnsignedWord amountToCopy = size.belowThan(oldSize) ? size : oldSize; - libc().memcpy(newBlock, oldInnerPtr, amountToCopy); - // Don't raw free the old block to avoid libc returning the same address later. - } - return true; - } - return false; + public static RuntimeSupport.Hook shutdownHook() { + return isFirstIsolate -> NativeMemoryTracking.singleton().printStatistics(); } - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - private void recordPreInitAlloc(Pointer newOuterPointer, UnsignedWord size, int flag, ReturnAddress returnAddress) { - if (newOuterPointer.isNull()) { - returnAddress.set(WordFactory.nullPointer()); - return; - } - Pointer newInnerPtr = recordMalloc0(newOuterPointer, size, flag); - preInitTable.putIfAbsent(newInnerPtr); - returnAddress.set(newInnerPtr); - } + public void printStatistics() { + // TEMP (chaeubl): disable for now + if (!SubstrateOptions.DiagnosticDetails.getValue().isEmpty()) { + System.out.println(); + System.out.println("Native memory tracking"); + System.out.println(" Total used memory: " + mallocMemorySnapshot.getTotalInfo().getUsed() + " bytes"); + System.out.println(" Total alive allocations: " + mallocMemorySnapshot.getTotalInfo().getCount()); - /** Special handling is needed for pre-init mallocs if NMT is disabled post-init. */ - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - public static boolean handlePreInitFrees(Pointer innerPtr) { - if (HasNmtSupport.get()) { - return get().handlePreInitFrees0(innerPtr); - } - return false; - } + for (int i = 0; i < NmtCategory.values().length; i++) { + String name = NmtCategory.values()[i].getName(); + NmtMallocMemoryInfo info = mallocMemorySnapshot.getInfoByCategory(i); - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - private boolean handlePreInitFrees0(Pointer innerPtr) { - if (!initialized) { - lockNoTransition(); - try { - if (!initialized) { - deaccountMalloc(innerPtr); - preInitTable.remove(innerPtr); - libc().free(innerPtr.subtract(getHeaderSize0())); - return true; - } - } finally { - unlock(); + System.out.println(" " + name + " used memory: " + info.getUsed() + " bytes"); + System.out.println(" " + name + " alive allocations: " + info.getCount()); } } - - // Post init. - if (!enabled && preInitTable.get(innerPtr).isNonNull()) { - // If NMT is now disabled and we're dealing with a pre-init block. - // There is a header from pre-init time, but tracking has since been disabled. - // Do nothing more. Don't Raw free the old block to avoid libc returning the same - // address later. - return true; - - } - // Either NMT is still enabled or we're dealing with a block allocated after initialization. - // We can proceed with tracking normally. - return false; - } - - @Uninterruptible(reason = "Locking without transition requires that the whole critical section is uninterruptible.", callerMustBe = true) - private void lockNoTransition() { - JavaSpinLockUtils.lockNoTransition(this, LOCK_OFFSET); - } - - @Uninterruptible(reason = "Locking without transition requires that the whole critical section is uninterruptible.", callerMustBe = true) - private void unlock() { - JavaSpinLockUtils.unlock(this, LOCK_OFFSET); - } - - public static RuntimeSupport.Hook shutdownHook() { - return isFirstIsolate -> { - printStats(); - }; - } - - public static RuntimeSupport.Hook startupHook() { - return isFirstIsolate -> { - initialize(SubstrateOptions.NativeMemoryTracking.getValue()); - }; } } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/NmtFlag.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/NmtCategory.java similarity index 65% rename from substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/NmtFlag.java rename to substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/NmtCategory.java index 7f3a371117ca..5cabf99520d3 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/NmtFlag.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/NmtCategory.java @@ -27,24 +27,37 @@ import com.oracle.svm.core.Uninterruptible; -/** These category flag names match their counterparts in Hotspot. */ -public enum NmtFlag { - mtJavaHeap("Java Heap"), - mtThread("Thread"), - mtThreadStack("Thread Stack"), - mtServiceability("Serviceability"), - mtGC("GC"), - mtInternal("Internal"), // Memory used by VM, outside other categories - mtCode("Code"), - mtOther("Other"), // Memory not used by VM (Unsafe) - mtNMT("Native Memory Tracking"), // Memory used by NMT itself - mtTest("Test"), // Test type for verifying NMT - mtTracing("Tracing"), // JFR - mtNone("Unknown"); // This is the default category +/** Categories for native memory tracking. */ +public enum NmtCategory { + /** JIT compiler. */ + Compiler("Compiler"), + /** JIT compiled code. */ + Code("Code"), + /** Garbage collector. */ + GC("GC"), + /** Heap dumping infrastructure. */ + HeapDump("Heap Dump"), + /** Java Flight Recorder. */ + JFR("JFR"), + /** Java Native Interface. */ + JNI("JNI"), + /** JVM stat / perf data. */ + JvmStat("jvmstat"), + /** NMT itself. */ + NMT("Native Memory Tracking"), + /** Profile-guided optimizations. */ + PGO("PGO"), + /** Threading. */ + Threading("Threading"), + /** Memory allocated via Unsafe. */ + Unsafe("Unsafe"), + + /** Some other VM internal reason - avoid if possible, better to add a new category. */ + Other("Other"); private final String name; - NmtFlag(String name) { + NmtCategory(String name) { this.name = name; } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/NmtFeature.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/NmtFeature.java index ca6d96b244a6..9b7786681fe4 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/NmtFeature.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/NmtFeature.java @@ -26,9 +26,9 @@ package com.oracle.svm.core.nmt; -import com.oracle.svm.core.VMInspectionOptions; import org.graalvm.nativeimage.ImageSingletons; +import com.oracle.svm.core.VMInspectionOptions; import com.oracle.svm.core.feature.AutomaticallyRegisteredFeature; import com.oracle.svm.core.feature.InternalFeature; import com.oracle.svm.core.jdk.RuntimeSupport; @@ -37,11 +37,7 @@ public class NmtFeature implements InternalFeature { @Override public boolean isInConfiguration(IsInConfigurationAccess access) { - return isInConfiguration(); - } - - public static boolean isInConfiguration() { - return VMInspectionOptions.hasNmtSupport(); + return VMInspectionOptions.hasNativeMemoryTrackingSupport(); } @Override @@ -51,10 +47,6 @@ public void afterRegistration(AfterRegistrationAccess access) { @Override public void beforeAnalysis(BeforeAnalysisAccess access) { - RuntimeSupport runtime = RuntimeSupport.getRuntimeSupport(); - - runtime.addShutdownHook(NativeMemoryTracking.shutdownHook()); - runtime.addStartupHook(NativeMemoryTracking.startupHook()); + RuntimeSupport.getRuntimeSupport().addShutdownHook(NativeMemoryTracking.shutdownHook()); } - } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/MallocHeader.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/NmtMallocHeader.java similarity index 72% rename from substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/MallocHeader.java rename to substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/NmtMallocHeader.java index f0b6929fd15b..5f2462bb07b7 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/MallocHeader.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/NmtMallocHeader.java @@ -29,23 +29,30 @@ import org.graalvm.nativeimage.c.struct.RawField; import org.graalvm.nativeimage.c.struct.RawStructure; import org.graalvm.word.PointerBase; +import org.graalvm.word.UnsignedWord; /** - * A "malloc header" is used to cache metadata about the native allocation (calloc/realloc/malloc). - * To do this, a small amount of additional space is requested contiguous to the user allocation. - * This metadata is later used to update the memory tracking once the block is freed. + * A "malloc header" stores metadata about the native allocation (malloc/calloc/realloc). To do + * this, a small amount of additional space is requested contiguous to the user allocation. This + * metadata is used to update the memory tracking once the block is freed. */ @RawStructure -public interface MallocHeader extends PointerBase { +public interface NmtMallocHeader extends PointerBase { @RawField - long getSize(); + UnsignedWord getAllocationSize(); @RawField - void setSize(long value); + void setAllocationSize(UnsignedWord value); @RawField - int getFlag(); + int getCategory(); @RawField - void setFlag(int value); + void setCategory(int value); + + @RawField + int getMagic(); + + @RawField + void setMagic(int value); } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/MallocMemoryInfo.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/NmtMallocMemoryInfo.java similarity index 81% rename from substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/MallocMemoryInfo.java rename to substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/NmtMallocMemoryInfo.java index c2159078129a..7a54c3b491ca 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/MallocMemoryInfo.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/NmtMallocMemoryInfo.java @@ -26,38 +26,36 @@ package com.oracle.svm.core.nmt; -import com.oracle.svm.core.jdk.UninterruptibleUtils.AtomicLong; +import org.graalvm.nativeimage.Platform; +import org.graalvm.nativeimage.Platforms; +import org.graalvm.word.UnsignedWord; import com.oracle.svm.core.Uninterruptible; -import org.graalvm.word.UnsignedWord; -import org.graalvm.nativeimage.Platforms; -import org.graalvm.nativeimage.Platform; +import com.oracle.svm.core.jdk.UninterruptibleUtils.AtomicLong; -class MallocMemoryInfo { - private AtomicLong count; - private AtomicLong size; +class NmtMallocMemoryInfo { + private final AtomicLong count = new AtomicLong(0); + private final AtomicLong used = new AtomicLong(0); @Platforms(Platform.HOSTED_ONLY.class) - MallocMemoryInfo() { - count = new AtomicLong(0); - size = new AtomicLong(0); + NmtMallocMemoryInfo() { } @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - void recordMalloc(UnsignedWord allocationSize) { + void track(UnsignedWord allocationSize) { count.incrementAndGet(); - size.addAndGet(allocationSize.rawValue()); + used.addAndGet(allocationSize.rawValue()); } @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - void deaccountMalloc(long allocationSize) { + void untrack(UnsignedWord allocationSize) { long lastCount = count.decrementAndGet(); - long lastSize = size.addAndGet(-allocationSize); + long lastSize = used.addAndGet(-allocationSize.rawValue()); assert lastSize >= 0 && lastCount >= 0; } - long getSize() { - return size.get(); + long getUsed() { + return used.get(); } long getCount() { diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/MallocMemorySnapshot.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/NmtMallocMemorySnapshot.java similarity index 70% rename from substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/MallocMemorySnapshot.java rename to substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/NmtMallocMemorySnapshot.java index 07336bf784fd..d5a8bca94a31 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/MallocMemorySnapshot.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/NmtMallocMemorySnapshot.java @@ -26,31 +26,37 @@ package com.oracle.svm.core.nmt; -import com.oracle.svm.core.Uninterruptible; -import org.graalvm.nativeimage.Platforms; import org.graalvm.nativeimage.Platform; +import org.graalvm.nativeimage.Platforms; + +import com.oracle.svm.core.Uninterruptible; -public class MallocMemorySnapshot { - private MallocMemoryInfo[] categories; - private MallocMemoryInfo total; +class NmtMallocMemorySnapshot { + private final NmtMallocMemoryInfo[] categories; + private final NmtMallocMemoryInfo total; @Platforms(Platform.HOSTED_ONLY.class) - MallocMemorySnapshot() { - total = new MallocMemoryInfo(); - categories = new MallocMemoryInfo[NmtFlag.values().length]; + NmtMallocMemorySnapshot() { + total = new NmtMallocMemoryInfo(); + categories = new NmtMallocMemoryInfo[NmtCategory.values().length]; for (int i = 0; i < categories.length; i++) { - categories[i] = new MallocMemoryInfo(); + categories[i] = new NmtMallocMemoryInfo(); } } @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - MallocMemoryInfo getInfoByCategory(int flag) { - assert flag < categories.length; - return categories[flag]; + NmtMallocMemoryInfo getInfoByCategory(NmtCategory category) { + return getInfoByCategory(category.ordinal()); + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + NmtMallocMemoryInfo getInfoByCategory(int category) { + assert category < categories.length; + return categories[category]; } @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - public MallocMemoryInfo getTotalInfo() { + NmtMallocMemoryInfo getTotalInfo() { return total; } } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/NmtTestFeature.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/NmtTestFeature.java deleted file mode 100644 index cedbdf21a0b3..000000000000 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/NmtTestFeature.java +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Copyright (c) 2023, 2023, Oracle and/or its affiliates. All rights reserved. - * Copyright (c) 2023, 2023, Red Hat Inc. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. Oracle designates this - * particular file as subject to the "Classpath" exception as provided - * by Oracle in the LICENSE file that accompanied this code. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - */ - -package com.oracle.svm.core.nmt; - -import org.graalvm.nativeimage.hosted.Feature; -import org.graalvm.nativeimage.ImageSingletons; - -/** This feature is required to omit the startup and shutdown hooks. */ -class NmtTestFeature implements Feature { - @Override - public void afterRegistration(AfterRegistrationAccess access) { - ImageSingletons.add(NativeMemoryTracking.class, new NativeMemoryTracking()); - } - - @Override - public boolean isInConfiguration(IsInConfigurationAccess access) { - return !NmtFeature.isInConfiguration(); - } -} diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/PreInitTable.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/PreInitTable.java deleted file mode 100644 index 8b4d909cc8a5..000000000000 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/PreInitTable.java +++ /dev/null @@ -1,108 +0,0 @@ -/* - * Copyright (c) 2023, 2023, Oracle and/or its affiliates. All rights reserved. - * Copyright (c) 2023, 2023, Red Hat Inc. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. Oracle designates this - * particular file as subject to the "Classpath" exception as provided - * by Oracle in the LICENSE file that accompanied this code. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - */ - -package com.oracle.svm.core.nmt; - -import com.oracle.svm.core.Uninterruptible; -import com.oracle.svm.core.UnmanagedMemoryUtil; -import com.oracle.svm.core.collections.UninterruptibleEntry; -import com.oracle.svm.core.collections.AbstractUninterruptibleHashtable; -import com.oracle.svm.core.headers.LibCSupport; -import org.graalvm.nativeimage.c.struct.SizeOf; -import org.graalvm.word.Pointer; -import org.graalvm.word.UnsignedWord; -import org.graalvm.word.WordFactory; - -import org.graalvm.nativeimage.ImageSingletons; -import org.graalvm.nativeimage.StackValue; -import com.oracle.svm.core.jdk.UninterruptibleUtils; - -/** - * This table stores addresses of NMT pre-init malloc blocks. These blocks will always contain - * headers. It is important that this class only uses raw malloc/free to avoid recursive behaviour. - * This table is read-only after NMT is initialized to mitigate the cost of synchronization. - */ -class PreInitTable extends AbstractUninterruptibleHashtable { - - @Override - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - protected UninterruptibleEntry[] createTable(int length) { - return new UninterruptibleEntry[length]; - } - - @Override - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - protected boolean isEqual(UninterruptibleEntry a, UninterruptibleEntry b) { - return a.getHash() == b.getHash(); - } - - /** This override is necessary to use LibC directly. */ - @Override - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - protected UninterruptibleEntry copyToHeap(UninterruptibleEntry valueOnStack) { - UnsignedWord size = SizeOf.unsigned(UninterruptibleEntry.class); - UninterruptibleEntry pointerOnHeap = ImageSingletons.lookup(LibCSupport.class).malloc(size); - if (pointerOnHeap.isNonNull()) { - NativeMemoryTracking.recordMallocWithoutHeader(size, NmtFlag.mtNMT.ordinal()); - UnmanagedMemoryUtil.copy((Pointer) valueOnStack, (Pointer) pointerOnHeap, SizeOf.unsigned(UninterruptibleEntry.class)); - return pointerOnHeap; - } - return WordFactory.nullPointer(); - } - - /** - * Since these native memory hashtable nodes don't have headers, this is necessary to avoid - * re-entering NMT code. Use LibC directly. - */ - @Override - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - protected void free(UninterruptibleEntry entry) { - size--; - ImageSingletons.lookup(LibCSupport.class).free(entry); - NativeMemoryTracking.deaccountMallocWithoutHeader(SizeOf.unsigned(UninterruptibleEntry.class).rawValue(), NmtFlag.mtNMT.ordinal()); - } - - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - public void remove(Pointer ptr) { - UninterruptibleEntry entry = StackValue.get(UninterruptibleEntry.class); - entry.setHash(UninterruptibleUtils.Long.hashCode(ptr.rawValue())); - remove(entry); - } - - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - public boolean putIfAbsent(Pointer ptr) { - UninterruptibleEntry entry = StackValue.get(UninterruptibleEntry.class); - entry.setHash(UninterruptibleUtils.Long.hashCode(ptr.rawValue())); - return putIfAbsent(entry); - } - - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - public UninterruptibleEntry get(Pointer ptr) { - UninterruptibleEntry entry = StackValue.get(UninterruptibleEntry.class); - entry.setHash(UninterruptibleUtils.Long.hashCode(ptr.rawValue())); - return get(entry); - } -} diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/ReturnAddress.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/ReturnAddress.java deleted file mode 100644 index e8820ecc8e57..000000000000 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/ReturnAddress.java +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Copyright (c) 2023, 2023, Oracle and/or its affiliates. All rights reserved. - * Copyright (c) 2023, 2023, Red Hat Inc. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. Oracle designates this - * particular file as subject to the "Classpath" exception as provided - * by Oracle in the LICENSE file that accompanied this code. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - */ - -package com.oracle.svm.core.nmt; - -import org.graalvm.nativeimage.c.struct.RawField; -import org.graalvm.nativeimage.c.struct.RawStructure; - -import org.graalvm.word.Pointer; -import org.graalvm.word.PointerBase; - -@RawStructure -public interface ReturnAddress extends PointerBase { - @RawField - Pointer get(); - - @RawField - void set(Pointer value); -} diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/os/AbstractRawFileOperationSupport.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/os/AbstractRawFileOperationSupport.java index af9d461fb5c1..753e63da36d0 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/os/AbstractRawFileOperationSupport.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/os/AbstractRawFileOperationSupport.java @@ -26,8 +26,6 @@ import java.io.File; -import jdk.graal.compiler.api.replacements.Fold; -import jdk.graal.compiler.word.Word; import org.graalvm.nativeimage.ImageSingletons; import org.graalvm.nativeimage.Platform; import org.graalvm.nativeimage.Platforms; @@ -41,6 +39,9 @@ import com.oracle.svm.core.hub.LayoutEncoding; import com.oracle.svm.core.snippets.KnownIntrinsics; +import jdk.graal.compiler.api.replacements.Fold; +import jdk.graal.compiler.word.Word; + public abstract class AbstractRawFileOperationSupport implements RawFileOperationSupport { private final boolean useNativeByteOrder; diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/os/BufferedFileOperationSupport.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/os/BufferedFileOperationSupport.java index f2638b6de817..c96df4dd61cb 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/os/BufferedFileOperationSupport.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/os/BufferedFileOperationSupport.java @@ -26,15 +26,12 @@ import java.nio.ByteOrder; -import jdk.graal.compiler.api.replacements.Fold; -import jdk.graal.compiler.word.Word; import org.graalvm.nativeimage.ImageSingletons; import org.graalvm.nativeimage.Platform; import org.graalvm.nativeimage.Platforms; import org.graalvm.nativeimage.c.struct.RawField; import org.graalvm.nativeimage.c.struct.RawStructure; import org.graalvm.nativeimage.c.struct.SizeOf; -import org.graalvm.nativeimage.impl.UnmanagedMemorySupport; import org.graalvm.word.Pointer; import org.graalvm.word.PointerBase; import org.graalvm.word.UnsignedWord; @@ -47,10 +44,15 @@ import com.oracle.svm.core.hub.DynamicHub; import com.oracle.svm.core.hub.LayoutEncoding; import com.oracle.svm.core.jdk.JavaLangSubstitutions; +import com.oracle.svm.core.memory.NullableNativeMemory; +import com.oracle.svm.core.nmt.NmtCategory; import com.oracle.svm.core.os.BufferedFileOperationSupport.BufferedFileOperationSupportHolder; import com.oracle.svm.core.os.RawFileOperationSupport.RawFileDescriptor; import com.oracle.svm.core.snippets.KnownIntrinsics; +import jdk.graal.compiler.api.replacements.Fold; +import jdk.graal.compiler.word.Word; + /** * Provides buffered, OS-independent operations on files. Most of the code is implemented in a way * that it can be used from uninterruptible code. @@ -99,7 +101,7 @@ protected BufferedFileOperationSupport(boolean useNativeByteOrder) { * Returns a null pointer otherwise. */ @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - public BufferedFile allocate(RawFileDescriptor fd) { + public BufferedFile allocate(RawFileDescriptor fd, NmtCategory nmtCategory) { if (!rawFiles().isValid(fd)) { return WordFactory.nullPointer(); } @@ -110,7 +112,7 @@ public BufferedFile allocate(RawFileDescriptor fd) { /* Use a single allocation for the struct and the corresponding buffer. */ UnsignedWord totalSize = SizeOf.unsigned(BufferedFile.class).add(WordFactory.unsigned(BUFFER_SIZE)); - BufferedFile result = ImageSingletons.lookup(UnmanagedMemorySupport.class).malloc(totalSize); + BufferedFile result = NullableNativeMemory.malloc(totalSize, nmtCategory); if (result.isNull()) { return WordFactory.nullPointer(); } @@ -127,7 +129,7 @@ public BufferedFile allocate(RawFileDescriptor fd) { */ @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void free(BufferedFile f) { - ImageSingletons.lookup(UnmanagedMemorySupport.class).free(f); + NullableNativeMemory.free(f); } /** diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/os/RawFileOperationSupport.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/os/RawFileOperationSupport.java index 139af6afa1b6..5ce50b5673fd 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/os/RawFileOperationSupport.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/os/RawFileOperationSupport.java @@ -26,17 +26,18 @@ import java.io.File; -import jdk.graal.compiler.api.replacements.Fold; import org.graalvm.nativeimage.ImageSingletons; import org.graalvm.nativeimage.c.type.CCharPointer; -import org.graalvm.nativeimage.impl.UnmanagedMemorySupport; import org.graalvm.word.Pointer; import org.graalvm.word.UnsignedWord; import org.graalvm.word.WordBase; import com.oracle.svm.core.Uninterruptible; +import com.oracle.svm.core.memory.UntrackedNullableNativeMemory; import com.oracle.svm.core.os.AbstractRawFileOperationSupport.RawFileOperationSupportHolder; +import jdk.graal.compiler.api.replacements.Fold; + /** * Provides unbuffered, OS-independent operations on files. Most of the code is implemented in a way * that it can be used from uninterruptible code. @@ -75,7 +76,7 @@ static RawFileOperationSupport nativeByteOrder() { /** * Tries to allocate a platform-dependent C string for the given path. Note that the returned * value needs to be freed manually once it is no longer needed (see - * {@link UnmanagedMemorySupport#free}). + * {@link UntrackedNullableNativeMemory#free}). * * @return If the allocation is successful, a non-null value is returned. */ diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/sampler/SamplerBufferPool.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/sampler/SamplerBufferPool.java index a7d2221ff925..b5f9643c1ef0 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/sampler/SamplerBufferPool.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/sampler/SamplerBufferPool.java @@ -28,7 +28,6 @@ import org.graalvm.nativeimage.ImageSingletons; import org.graalvm.nativeimage.Platform; import org.graalvm.nativeimage.Platforms; -import org.graalvm.nativeimage.impl.UnmanagedMemorySupport; import org.graalvm.word.UnsignedWord; import org.graalvm.word.WordFactory; @@ -37,6 +36,8 @@ import com.oracle.svm.core.jfr.SubstrateJVM; import com.oracle.svm.core.jfr.sampler.JfrExecutionSampler; import com.oracle.svm.core.locks.VMMutex; +import com.oracle.svm.core.memory.NullableNativeMemory; +import com.oracle.svm.core.nmt.NmtCategory; /** * Keeps track of {@link #availableBuffers available} and {@link #fullBuffers full} buffers. If @@ -160,7 +161,7 @@ private SamplerBuffer tryAllocateBuffer0() { UnsignedWord headerSize = SamplerBufferAccess.getHeaderSize(); UnsignedWord dataSize = WordFactory.unsigned(SubstrateJVM.getThreadLocal().getThreadLocalBufferSize()); - SamplerBuffer result = ImageSingletons.lookup(UnmanagedMemorySupport.class).malloc(headerSize.add(dataSize)); + SamplerBuffer result = NullableNativeMemory.malloc(headerSize.add(dataSize), NmtCategory.JFR); if (result.isNonNull()) { bufferCount++; result.setSize(dataSize); @@ -183,7 +184,7 @@ private boolean popAndFree() { @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private void free(SamplerBuffer buffer) { - ImageSingletons.lookup(UnmanagedMemorySupport.class).free(buffer); + NullableNativeMemory.free(buffer); bufferCount--; } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/PlatformThreads.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/PlatformThreads.java index 344c6895b28a..f8d33a71ea9a 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/PlatformThreads.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/PlatformThreads.java @@ -56,7 +56,6 @@ import org.graalvm.nativeimage.ObjectHandles; import org.graalvm.nativeimage.Platform; import org.graalvm.nativeimage.Platforms; -import org.graalvm.nativeimage.UnmanagedMemory; import org.graalvm.nativeimage.c.function.CEntryPoint; import org.graalvm.nativeimage.c.function.CEntryPointLiteral; import org.graalvm.nativeimage.c.function.CFunctionPointer; @@ -95,7 +94,9 @@ import com.oracle.svm.core.jfr.HasJfrSupport; import com.oracle.svm.core.locks.VMMutex; import com.oracle.svm.core.log.Log; +import com.oracle.svm.core.memory.NativeMemory; import com.oracle.svm.core.monitor.MonitorSupport; +import com.oracle.svm.core.nmt.NmtCategory; import com.oracle.svm.core.nodes.CFunctionEpilogueNode; import com.oracle.svm.core.nodes.CFunctionPrologueNode; import com.oracle.svm.core.stack.StackFrameVisitor; @@ -714,14 +715,14 @@ protected T prepareStart(Thread thread, int startDat T startData = WordFactory.nullPointer(); ObjectHandle threadHandle = WordFactory.zero(); try { - startData = UnmanagedMemory.malloc(startDataSize); + startData = NativeMemory.malloc(startDataSize, NmtCategory.Threading); threadHandle = ObjectHandles.getGlobal().create(thread); startData.setIsolate(CurrentIsolate.getIsolate()); startData.setThreadHandle(threadHandle); } catch (Throwable e) { if (startData.isNonNull()) { - UnmanagedMemory.free(startData); + freeStartData(startData); } if (threadHandle.notEqual(WordFactory.zero())) { ObjectHandles.getGlobal().destroy(threadHandle); @@ -779,7 +780,7 @@ private static void decrementNonDaemonThreadsAndNotify() { } protected static void freeStartData(ThreadStartData startData) { - UnmanagedMemory.free(startData); + NativeMemory.free(startData); } void startThread(Thread thread, long stackSize) { diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/VMThreads.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/VMThreads.java index 5e28857a0c7a..db22367483b3 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/VMThreads.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/VMThreads.java @@ -32,7 +32,6 @@ import org.graalvm.nativeimage.IsolateThread; import org.graalvm.nativeimage.c.function.CFunction; import org.graalvm.nativeimage.c.type.CCharPointer; -import org.graalvm.nativeimage.impl.UnmanagedMemorySupport; import org.graalvm.word.ComparableWord; import org.graalvm.word.Pointer; import org.graalvm.word.PointerBase; @@ -53,7 +52,7 @@ import com.oracle.svm.core.locks.VMLockSupport; import com.oracle.svm.core.locks.VMMutex; import com.oracle.svm.core.log.Log; -import com.oracle.svm.core.nmt.NmtFlag; +import com.oracle.svm.core.memory.UntrackedNullableNativeMemory; import com.oracle.svm.core.nodes.CFunctionEpilogueNode; import com.oracle.svm.core.nodes.CFunctionPrologueNode; import com.oracle.svm.core.threadlocal.FastThreadLocal; @@ -255,7 +254,7 @@ public IsolateThread allocateIsolateThread(int isolateThreadSize) { UnsignedWord alignment = WordFactory.unsigned(64); UnsignedWord memorySize = WordFactory.unsigned(isolateThreadSize).add(alignment); - Pointer memory = ImageSingletons.lookup(UnmanagedMemorySupport.class).calloc(memorySize, NmtFlag.mtThread.ordinal()); + Pointer memory = UntrackedNullableNativeMemory.calloc(memorySize); if (memory.isNull()) { return WordFactory.nullPointer(); } @@ -275,7 +274,7 @@ public void freeCurrentIsolateThread() { @Uninterruptible(reason = "Thread state no longer set up.") protected void freeIsolateThread(IsolateThread thread) { Pointer memory = unalignedIsolateThreadMemoryTL.get(thread); - ImageSingletons.lookup(UnmanagedMemorySupport.class).free(memory); + UntrackedNullableNativeMemory.free(memory); } /** diff --git a/substratevm/src/com.oracle.svm.graal/src/com/oracle/svm/graal/isolated/IsolatedReferenceAdjuster.java b/substratevm/src/com.oracle.svm.graal/src/com/oracle/svm/graal/isolated/IsolatedReferenceAdjuster.java index 453fbe54c1cf..664db15b0826 100644 --- a/substratevm/src/com.oracle.svm.graal/src/com/oracle/svm/graal/isolated/IsolatedReferenceAdjuster.java +++ b/substratevm/src/com.oracle.svm.graal/src/com/oracle/svm/graal/isolated/IsolatedReferenceAdjuster.java @@ -25,7 +25,6 @@ package com.oracle.svm.graal.isolated; import org.graalvm.nativeimage.ObjectHandle; -import org.graalvm.nativeimage.UnmanagedMemory; import org.graalvm.nativeimage.c.struct.SizeOf; import org.graalvm.word.Pointer; import org.graalvm.word.PointerBase; @@ -38,7 +37,9 @@ import com.oracle.svm.core.code.ReferenceAdjuster; import com.oracle.svm.core.config.ConfigurationValues; import com.oracle.svm.core.handles.ThreadLocalHandles; +import com.oracle.svm.core.memory.NativeMemory; import com.oracle.svm.core.meta.DirectSubstrateObjectConstant; +import com.oracle.svm.core.nmt.NmtCategory; import com.oracle.svm.core.util.VMError; import jdk.vm.ci.meta.JavaConstant; @@ -98,9 +99,9 @@ public void setConstantTargetAt(PointerBase address, int length, JavaConstant co @Override @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - public NonmovableObjectArray copyOfObjectArray(T[] source) { + public NonmovableObjectArray copyOfObjectArray(T[] source, NmtCategory nmtCategory) { @SuppressWarnings("unchecked") - NonmovableObjectArray copy = NonmovableArrays.createObjectArray((Class) source.getClass(), source.length); + NonmovableObjectArray copy = NonmovableArrays.createObjectArray((Class) source.getClass(), source.length, nmtCategory); for (int i = 0; i < source.length; i++) { setObjectInArray(copy, i, source[i]); } @@ -114,14 +115,12 @@ public boolean isFinished() { } public ForeignIsolateReferenceAdjusterData exportData() { - ForeignIsolateReferenceAdjusterData data = UnmanagedMemory.malloc(SizeOf.get(ForeignIsolateReferenceAdjusterData.class)); + ForeignIsolateReferenceAdjusterData data = NativeMemory.malloc(SizeOf.get(ForeignIsolateReferenceAdjusterData.class), NmtCategory.Compiler); data.setCount(count); data.setAddresses(addresses); data.setHandles(handles); - NonmovableArrays.untrackUnmanagedArray(addresses); addresses = WordFactory.nullPointer(); - NonmovableArrays.untrackUnmanagedArray(handles); handles = WordFactory.nullPointer(); count = 0; @@ -129,9 +128,6 @@ public ForeignIsolateReferenceAdjusterData exportData() { } public static void adjustAndDispose(ForeignIsolateReferenceAdjusterData data, ThreadLocalHandles handleSet) { - NonmovableArrays.trackUnmanagedArray(data.getAddresses()); - NonmovableArrays.trackUnmanagedArray(data.getHandles()); - int count = data.getCount(); for (int i = 0; i < count; i++) { @SuppressWarnings("unchecked") @@ -143,7 +139,7 @@ public static void adjustAndDispose(ForeignIsolateRefer NonmovableArrays.releaseUnmanagedArray(data.getAddresses()); NonmovableArrays.releaseUnmanagedArray(data.getHandles()); - UnmanagedMemory.free(data); + NativeMemory.free(data); } @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) @@ -166,8 +162,8 @@ private void growIfFull() { int oldSize = addresses.isNonNull() ? NonmovableArrays.lengthOf(addresses) : 0; if (count == oldSize) { int newSize = (oldSize != 0) ? (2 * oldSize) : 32; - NonmovableArray newAddresses = NonmovableArrays.createWordArray(newSize); - NonmovableArray newHandles = NonmovableArrays.createWordArray(newSize); + NonmovableArray newAddresses = NonmovableArrays.createWordArray(newSize, NmtCategory.Compiler); + NonmovableArray newHandles = NonmovableArrays.createWordArray(newSize, NmtCategory.Compiler); if (addresses.isNonNull()) { NonmovableArrays.arraycopy(addresses, 0, newAddresses, 0, oldSize); NonmovableArrays.releaseUnmanagedArray(addresses); diff --git a/substratevm/src/com.oracle.svm.graal/src/com/oracle/svm/graal/isolated/IsolatedRuntimeCodeInstaller.java b/substratevm/src/com.oracle.svm.graal/src/com/oracle/svm/graal/isolated/IsolatedRuntimeCodeInstaller.java index 9e1bb36ba2f5..2ad0fd3830fd 100644 --- a/substratevm/src/com.oracle.svm.graal/src/com/oracle/svm/graal/isolated/IsolatedRuntimeCodeInstaller.java +++ b/substratevm/src/com.oracle.svm.graal/src/com/oracle/svm/graal/isolated/IsolatedRuntimeCodeInstaller.java @@ -24,10 +24,7 @@ */ package com.oracle.svm.graal.isolated; -import jdk.graal.compiler.code.CompilationResult; -import jdk.graal.compiler.core.common.CompilationIdentifier; import org.graalvm.nativeimage.IsolateThread; -import org.graalvm.nativeimage.UnmanagedMemory; import org.graalvm.nativeimage.c.function.CEntryPoint; import org.graalvm.nativeimage.c.function.CodePointer; import org.graalvm.nativeimage.c.struct.SizeOf; @@ -40,11 +37,16 @@ import com.oracle.svm.core.code.RuntimeCodeInfoAccess; import com.oracle.svm.core.deopt.SubstrateInstalledCode; import com.oracle.svm.core.graal.meta.SharedRuntimeMethod; +import com.oracle.svm.core.memory.NativeMemory; import com.oracle.svm.core.meta.SharedMethod; +import com.oracle.svm.core.nmt.NmtCategory; import com.oracle.svm.graal.meta.RuntimeCodeInstaller; import com.oracle.svm.graal.meta.SubstrateInstalledCodeImpl; import com.oracle.svm.graal.meta.SubstrateMethod; +import jdk.graal.compiler.code.CompilationResult; +import jdk.graal.compiler.core.common.CompilationIdentifier; + public final class IsolatedRuntimeCodeInstaller extends RuntimeCodeInstaller { /** @@ -114,23 +116,23 @@ private CodeInstallInfo doPrepareInstall() { id = proxy.getHandle(); } - CodeInstallInfo installInfo = UnmanagedMemory.malloc(SizeOf.get(CodeInstallInfo.class)); + CodeInstallInfo installInfo = NativeMemory.malloc(SizeOf.get(CodeInstallInfo.class), NmtCategory.Compiler); installInfo.setCodeInfo(codeInfo); installInfo.setAdjusterData(adjuster.exportData()); installInfo.setCompilationId(id); - IsolatedRuntimeMethodInfoAccess.untrackInCurrentIsolate(installInfo.getCodeInfo()); + IsolatedRuntimeMethodInfoAccess.untrackInCurrentIsolate(installInfo); return installInfo; } private static void installPrepared(SharedMethod method, CodeInstallInfo installInfo, SubstrateInstalledCode installedCode) { - IsolatedRuntimeMethodInfoAccess.startTrackingInCurrentIsolate(installInfo.getCodeInfo()); + IsolatedRuntimeMethodInfoAccess.startTrackingInCurrentIsolate(installInfo); IsolatedReferenceAdjuster.adjustAndDispose(installInfo.getAdjusterData(), IsolatedCompileClient.get().getHandleSet()); installInfo.setAdjusterData(WordFactory.nullPointer()); doInstallPrepared(method, installInfo.getCodeInfo(), installedCode); - UnmanagedMemory.free(installInfo); + NativeMemory.free(installInfo); } private final IsolateThread targetIsolate; diff --git a/substratevm/src/com.oracle.svm.graal/src/com/oracle/svm/graal/isolated/IsolatedRuntimeMethodInfoAccess.java b/substratevm/src/com.oracle.svm.graal/src/com/oracle/svm/graal/isolated/IsolatedRuntimeMethodInfoAccess.java index 54330d27bcbf..a2dbf8c5cf63 100644 --- a/substratevm/src/com.oracle.svm.graal/src/com/oracle/svm/graal/isolated/IsolatedRuntimeMethodInfoAccess.java +++ b/substratevm/src/com.oracle.svm.graal/src/com/oracle/svm/graal/isolated/IsolatedRuntimeMethodInfoAccess.java @@ -25,25 +25,57 @@ package com.oracle.svm.graal.isolated; import com.oracle.svm.core.Uninterruptible; +import com.oracle.svm.core.VMInspectionOptions; import com.oracle.svm.core.c.NonmovableArray; import com.oracle.svm.core.c.NonmovableArrays; import com.oracle.svm.core.code.CodeInfo; import com.oracle.svm.core.code.InstalledCodeObserverSupport; import com.oracle.svm.core.code.RuntimeCodeInfoAccess; import com.oracle.svm.core.code.RuntimeCodeInfoMemory; +import com.oracle.svm.core.nmt.NativeMemoryTracking; final class IsolatedRuntimeMethodInfoAccess { - public static void startTrackingInCurrentIsolate(CodeInfo info) { - RuntimeCodeInfoAccess.forEachArray(info, TRACK_ACTION); + public static void startTrackingInCurrentIsolate(CodeInstallInfo installInfo) { + CodeInfo info = installInfo.getCodeInfo(); InstalledCodeObserverSupport.attachToCurrentIsolate(RuntimeCodeInfoAccess.getCodeObserverHandles(info)); RuntimeCodeInfoMemory.singleton().add(info); + + /* NonmovableArray tracking and native memory tracking. */ + RuntimeCodeInfoAccess.forEachArray(info, TRACK_ACTION); + + ForeignIsolateReferenceAdjusterData adjusterData = installInfo.getAdjusterData(); + NonmovableArrays.trackUnmanagedArray(adjusterData.getAddresses()); + NonmovableArrays.trackUnmanagedArray(adjusterData.getHandles()); + + if (VMInspectionOptions.hasNativeMemoryTrackingSupport()) { + NativeMemoryTracking.singleton().track(installInfo); + NativeMemoryTracking.singleton().track(installInfo.getAdjusterData()); + NativeMemoryTracking.singleton().track(installInfo.getCodeInfo()); + NativeMemoryTracking.singleton().track(adjusterData.getAddresses()); + NativeMemoryTracking.singleton().track(adjusterData.getHandles()); + } } - public static void untrackInCurrentIsolate(CodeInfo info) { + public static void untrackInCurrentIsolate(CodeInstallInfo installInfo) { + CodeInfo info = installInfo.getCodeInfo(); RuntimeCodeInfoMemory.singleton().remove(info); InstalledCodeObserverSupport.detachFromCurrentIsolate(RuntimeCodeInfoAccess.getCodeObserverHandles(info)); + + /* NonmovableArray tracking and native memory tracking. */ RuntimeCodeInfoAccess.forEachArray(info, UNTRACK_ACTION); + + ForeignIsolateReferenceAdjusterData adjusterData = installInfo.getAdjusterData(); + NonmovableArrays.untrackUnmanagedArray(adjusterData.getAddresses()); + NonmovableArrays.untrackUnmanagedArray(adjusterData.getHandles()); + + if (VMInspectionOptions.hasNativeMemoryTrackingSupport()) { + NativeMemoryTracking.singleton().untrack(installInfo); + NativeMemoryTracking.singleton().untrack(installInfo.getAdjusterData()); + NativeMemoryTracking.singleton().untrack(installInfo.getCodeInfo()); + NativeMemoryTracking.singleton().untrack(adjusterData.getAddresses()); + NativeMemoryTracking.singleton().untrack(adjusterData.getHandles()); + } } private static final RuntimeCodeInfoAccess.NonmovableArrayAction TRACK_ACTION = new RuntimeCodeInfoAccess.NonmovableArrayAction() { @@ -51,6 +83,9 @@ public static void untrackInCurrentIsolate(CodeInfo info) { @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void apply(NonmovableArray array) { NonmovableArrays.trackUnmanagedArray(array); + if (VMInspectionOptions.hasNativeMemoryTrackingSupport()) { + NativeMemoryTracking.singleton().track(array); + } } }; private static final RuntimeCodeInfoAccess.NonmovableArrayAction UNTRACK_ACTION = new RuntimeCodeInfoAccess.NonmovableArrayAction() { @@ -58,6 +93,9 @@ public void apply(NonmovableArray array) { @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void apply(NonmovableArray array) { NonmovableArrays.untrackUnmanagedArray(array); + if (VMInspectionOptions.hasNativeMemoryTrackingSupport()) { + NativeMemoryTracking.singleton().untrack(array); + } } }; diff --git a/substratevm/src/com.oracle.svm.test/src/META-INF/native-image/com.oracle.svm.test/native-image.properties b/substratevm/src/com.oracle.svm.test/src/META-INF/native-image/com.oracle.svm.test/native-image.properties index c72a60578e4a..e1470477f070 100644 --- a/substratevm/src/com.oracle.svm.test/src/META-INF/native-image/com.oracle.svm.test/native-image.properties +++ b/substratevm/src/com.oracle.svm.test/src/META-INF/native-image/com.oracle.svm.test/native-image.properties @@ -6,8 +6,7 @@ Args= \ --features=com.oracle.svm.test.NoProviderConstructorServiceLoaderTest$TestFeature \ --features=com.oracle.svm.test.NativeImageResourceUtils$TestFeature \ --features=com.oracle.svm.test.jfr.JfrTestFeature \ - --features=com.oracle.svm.core.nmt.NmtTestFeature \ --add-opens=java.base/java.lang=ALL-UNNAMED \ --add-exports=org.graalvm.nativeimage.base/com.oracle.svm.util=ALL-UNNAMED \ - --enable-monitoring=jvmstat,jfr,jmxserver,jmxclient \ + --enable-monitoring=jvmstat,jfr,jmxserver,jmxclient,nmt \ -J--enable-preview \ No newline at end of file diff --git a/substratevm/src/com.oracle.svm.test/src/com/oracle/svm/test/debug/CStructTests.java b/substratevm/src/com.oracle.svm.test/src/com/oracle/svm/test/debug/CStructTests.java index 84e0a7074b94..4bf39fa84729 100644 --- a/substratevm/src/com.oracle.svm.test/src/com/oracle/svm/test/debug/CStructTests.java +++ b/substratevm/src/com.oracle.svm.test/src/com/oracle/svm/test/debug/CStructTests.java @@ -24,7 +24,6 @@ */ package com.oracle.svm.test.debug; -import org.graalvm.nativeimage.UnmanagedMemory; import org.graalvm.nativeimage.c.CContext; import org.graalvm.nativeimage.c.struct.CField; import org.graalvm.nativeimage.c.struct.CFieldAddress; @@ -38,6 +37,8 @@ import org.graalvm.word.SignedWord; import com.oracle.svm.core.NeverInline; +import com.oracle.svm.core.memory.NativeMemory; +import com.oracle.svm.core.nmt.NmtCategory; import jdk.graal.compiler.api.directives.GraalDirectives; @@ -119,7 +120,7 @@ interface Weird extends PointerBase { ]) */ public static void weird() { - Weird wd = UnmanagedMemory.malloc(SizeOf.get(Weird.class)); + Weird wd = NativeMemory.malloc(SizeOf.get(Weird.class), NmtCategory.Other); wd.setf_short((short) 42); wd.setf_int(43); @@ -208,7 +209,7 @@ interface CompositeStruct extends PointerBase { ]) */ public static void composite() { - CompositeStruct cs = UnmanagedMemory.malloc(3 * SizeOf.get(CompositeStruct.class)); + CompositeStruct cs = NativeMemory.malloc(3 * SizeOf.get(CompositeStruct.class), NmtCategory.Other); cs.setC1((byte) 7); cs.setC3(13); cs.setC5((short) 32000); @@ -220,8 +221,8 @@ public static void composite() { } public static void mixedArguments() { - SimpleStruct ss1 = UnmanagedMemory.malloc(SizeOf.get(SimpleStruct.class)); - SimpleStruct2 ss2 = UnmanagedMemory.malloc(SizeOf.get(SimpleStruct2.class)); + SimpleStruct ss1 = NativeMemory.malloc(SizeOf.get(SimpleStruct.class), NmtCategory.Other); + SimpleStruct2 ss2 = NativeMemory.malloc(SizeOf.get(SimpleStruct2.class), NmtCategory.Other); String m1 = "a message in a bottle"; String m2 = "a ship in a bottle"; String m3 = "courage in a bottle"; @@ -251,6 +252,6 @@ public static void main(String[] args) { @NeverInline("Used as a hook to inspect the caller frame in GDB") static void free(PointerBase ptr) { - UnmanagedMemory.free(ptr); + NativeMemory.free(ptr); } } diff --git a/substratevm/src/com.oracle.svm.test/src/com/oracle/svm/test/nmt/NativeMemoryTrackingTests.java b/substratevm/src/com.oracle.svm.test/src/com/oracle/svm/test/nmt/NativeMemoryTrackingTests.java new file mode 100644 index 000000000000..3cbacd41288c --- /dev/null +++ b/substratevm/src/com.oracle.svm.test/src/com/oracle/svm/test/nmt/NativeMemoryTrackingTests.java @@ -0,0 +1,90 @@ +/* + * Copyright (c) 2023, 2023, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2023, 2023, Red Hat Inc. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +package com.oracle.svm.test.nmt; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import org.graalvm.word.Pointer; +import org.graalvm.word.WordFactory; +import org.junit.Test; + +import com.oracle.svm.core.memory.NativeMemory; +import com.oracle.svm.core.nmt.NativeMemoryTracking; +import com.oracle.svm.core.nmt.NmtCategory; + +public class NativeMemoryTrackingTests { + private static final int ALLOCATION_SIZE = 1024 * 16; + private static final int RESERVE_SIZE = ALLOCATION_SIZE; + private static final int REALLOC_SIZE = ALLOCATION_SIZE / 2; + + @Test + public void testMalloc() { + assertEquals(0, getUsedMemory()); + + Pointer ptr = NativeMemory.malloc(WordFactory.unsigned(ALLOCATION_SIZE), NmtCategory.Code); + assertEquals(ALLOCATION_SIZE, getUsedMemory()); + assertTrue(getUsedMemory() > 0); + + NativeMemory.free(ptr); + + assertEquals(0, getUsedMemory()); + } + + @Test + public void testCalloc() { + assertEquals(0, getUsedMemory()); + Pointer ptr = NativeMemory.calloc(WordFactory.unsigned(ALLOCATION_SIZE), NmtCategory.Code); + + assertEquals(ALLOCATION_SIZE, getUsedMemory()); + assertTrue(getUsedMemory() > 0); + + NativeMemory.free(ptr); + + assertEquals(0, getUsedMemory()); + } + + @Test + public void testRealloc() { + assertEquals(0, getUsedMemory()); + Pointer ptr = NativeMemory.malloc(WordFactory.unsigned(ALLOCATION_SIZE), NmtCategory.Code); + + assertEquals(getUsedMemory(), ALLOCATION_SIZE); + assertTrue(getUsedMemory() > 0); + + Pointer reallocPtr = NativeMemory.realloc(ptr, WordFactory.unsigned(REALLOC_SIZE), NmtCategory.Code); + + assertEquals(REALLOC_SIZE, getUsedMemory()); + + NativeMemory.free(reallocPtr); + assertEquals(0, getUsedMemory()); + } + + private static long getUsedMemory() { + return NativeMemoryTracking.singleton().getUsedMemory(NmtCategory.Code); + } +} diff --git a/substratevm/src/com.oracle.svm.test/src/com/oracle/svm/test/nmt/TestBasic.java b/substratevm/src/com.oracle.svm.test/src/com/oracle/svm/test/nmt/TestBasic.java deleted file mode 100644 index 4ae6149fe5cd..000000000000 --- a/substratevm/src/com.oracle.svm.test/src/com/oracle/svm/test/nmt/TestBasic.java +++ /dev/null @@ -1,122 +0,0 @@ -/* - * Copyright (c) 2023, 2023, Oracle and/or its affiliates. All rights reserved. - * Copyright (c) 2023, 2023, Red Hat Inc. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. Oracle designates this - * particular file as subject to the "Classpath" exception as provided - * by Oracle in the LICENSE file that accompanied this code. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - */ - -package com.oracle.svm.test.nmt; - -import com.oracle.svm.core.nmt.NmtFlag; - -import org.junit.BeforeClass; -import org.junit.Test; - -import com.oracle.svm.core.nmt.NativeMemoryTracking; -import org.graalvm.nativeimage.ImageSingletons; -import org.graalvm.nativeimage.impl.UnmanagedMemorySupport; -import org.graalvm.word.WordFactory; -import org.graalvm.word.Pointer; -import static org.junit.Assert.assertTrue; - -public class TestBasic { - private static final int ALLOCATION_SIZE = 1024 * 16; - private static final int RESERVE_SIZE = ALLOCATION_SIZE; - private static final int REALLOC_SIZE = ALLOCATION_SIZE / 2; - private static final int COMMIT_SIZE = RESERVE_SIZE / 2; - - /** - * This both initializes NMT and does some basic checks to verify pre-init allocations are - * handled. - */ - @BeforeClass - public static void setup() { - // Allocate some memory and check it's being tracked. - Pointer ptr = ImageSingletons.lookup(UnmanagedMemorySupport.class).malloc(WordFactory.unsigned(ALLOCATION_SIZE), NmtFlag.mtTest.ordinal()); - assertTrue(NativeMemoryTracking.getMallocByCategory(NmtFlag.mtTest) == ALLOCATION_SIZE); - - // Realloc previously allocated memory and check NMT has tracked it correctly - Pointer reallocPtr = ImageSingletons.lookup(UnmanagedMemorySupport.class).realloc(ptr, WordFactory.unsigned(REALLOC_SIZE), NmtFlag.mtTest.ordinal()); - assertTrue(NativeMemoryTracking.getMallocByCategory(NmtFlag.mtTest) == REALLOC_SIZE); - - // Free the memory and ensure the tracking is now zeroed. - ImageSingletons.lookup(UnmanagedMemorySupport.class).free(reallocPtr); - assertTrue(NativeMemoryTracking.getMallocByCategory(NmtFlag.mtTest) == 0); - - // Allocate a new block that will live across the initialization boundary - ptr = ImageSingletons.lookup(UnmanagedMemorySupport.class).malloc(WordFactory.unsigned(ALLOCATION_SIZE), NmtFlag.mtTest.ordinal()); - - // We must initialize NMT here so that other tests can use it. - NativeMemoryTracking.initialize(true); - - // Reallocate a block that has lived across the initialization boundary. Verify its been - // recorded. - reallocPtr = ImageSingletons.lookup(UnmanagedMemorySupport.class).realloc(ptr, WordFactory.unsigned(REALLOC_SIZE), NmtFlag.mtTest.ordinal()); - assertTrue(NativeMemoryTracking.getMallocByCategory(NmtFlag.mtTest) == REALLOC_SIZE); - - // Free the memory and verify we're back at zero. - ImageSingletons.lookup(UnmanagedMemorySupport.class).free(reallocPtr); - assertTrue(NativeMemoryTracking.getMallocByCategory(NmtFlag.mtTest) == 0); - } - - @Test - public void testMalloc() throws Throwable { - assertTrue("Test should start with no memory already allocated in the mtTest category.", NativeMemoryTracking.getMallocByCategory(NmtFlag.mtTest) == 0); - - Pointer ptr = ImageSingletons.lookup(UnmanagedMemorySupport.class).malloc(WordFactory.unsigned(ALLOCATION_SIZE), NmtFlag.mtTest.ordinal()); - assertTrue(NativeMemoryTracking.getMallocByCategory(NmtFlag.mtTest) == ALLOCATION_SIZE); - assertTrue(NativeMemoryTracking.getMallocByCategory(NmtFlag.mtNMT) > 0); - - ImageSingletons.lookup(UnmanagedMemorySupport.class).free(ptr); - - assertTrue("After freeing memory for test, mtTest category should have size 0.", NativeMemoryTracking.getMallocByCategory(NmtFlag.mtTest) == 0); - } - - @Test - public void testCalloc() throws Throwable { - assertTrue("Test should start with no memory already allocated in the mtTest category.", NativeMemoryTracking.getMallocByCategory(NmtFlag.mtTest) == 0); - Pointer ptr = ImageSingletons.lookup(UnmanagedMemorySupport.class).calloc(WordFactory.unsigned(ALLOCATION_SIZE), NmtFlag.mtTest.ordinal()); - - assertTrue(NativeMemoryTracking.getMallocByCategory(NmtFlag.mtTest) == ALLOCATION_SIZE); - assertTrue(NativeMemoryTracking.getMallocByCategory(NmtFlag.mtNMT) > 0); - - ImageSingletons.lookup(UnmanagedMemorySupport.class).free(ptr); - - assertTrue("After freeing memory for test, mtTest category should have size 0.", NativeMemoryTracking.getMallocByCategory(NmtFlag.mtTest) == 0); - } - - @Test - public void testRealloc() throws Throwable { - assertTrue("Test should start with no memory already allocated in the mtTest category.", NativeMemoryTracking.getMallocByCategory(NmtFlag.mtTest) == 0); - Pointer ptr = ImageSingletons.lookup(UnmanagedMemorySupport.class).malloc(WordFactory.unsigned(ALLOCATION_SIZE), NmtFlag.mtTest.ordinal()); - - assertTrue(NativeMemoryTracking.getMallocByCategory(NmtFlag.mtTest) == ALLOCATION_SIZE); - assertTrue(NativeMemoryTracking.getMallocByCategory(NmtFlag.mtNMT) > 0); - - Pointer reallocPtr = ImageSingletons.lookup(UnmanagedMemorySupport.class).realloc(ptr, WordFactory.unsigned(REALLOC_SIZE), NmtFlag.mtTest.ordinal()); - - assertTrue(NativeMemoryTracking.getMallocByCategory(NmtFlag.mtTest) == REALLOC_SIZE); - - ImageSingletons.lookup(UnmanagedMemorySupport.class).free(reallocPtr); - assertTrue("After freeing memory for test, mtTest category should have size 0.", NativeMemoryTracking.getMallocByCategory(NmtFlag.mtTest) == 0); - } -} From bdd3c69b2fff98ce4015561abf73447aa5a09ffb Mon Sep 17 00:00:00 2001 From: Christian Haeubl Date: Mon, 12 Feb 2024 10:41:57 +0100 Subject: [PATCH 6/7] Style and test fixes. --- .../com/oracle/svm/core/VMInspectionOptions.java | 11 ++++++++--- .../src/com/oracle/svm/core/headers/LibC.java | 2 -- .../oracle/svm/core/nmt/NativeMemoryTracking.java | 5 ++--- .../src/com/oracle/svm/test/debug/CStructTests.java | 13 ++++++------- .../svm/test/nmt/NativeMemoryTrackingTests.java | 1 - 5 files changed, 16 insertions(+), 16 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/VMInspectionOptions.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/VMInspectionOptions.java index b2f3d0d1e774..e5b7ad611a76 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/VMInspectionOptions.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/VMInspectionOptions.java @@ -40,6 +40,7 @@ import com.oracle.svm.core.option.APIOption; import com.oracle.svm.core.option.HostedOptionKey; import com.oracle.svm.core.option.LocatableMultiOptionValue; +import com.oracle.svm.core.option.RuntimeOptionKey; import com.oracle.svm.core.option.SubstrateOptionsParser; import com.oracle.svm.core.util.UserError; import com.oracle.svm.util.LogUtils; @@ -77,6 +78,13 @@ public final class VMInspectionOptions { public static final HostedOptionKey EnableMonitoringFeatures = new HostedOptionKey<>(LocatableMultiOptionValue.Strings.buildWithCommaDelimiter(), VMInspectionOptions::validateEnableMonitoringFeatures); + @Option(help = "Dumps all runtime compiled methods on SIGUSR2.", type = OptionType.User) // + public static final HostedOptionKey DumpRuntimeCompilationOnSignal = new HostedOptionKey<>(false); + + // TEMP (chaeubl): change default to true. + @Option(help = "Print native memory tracking statistics on shutdown.", type = OptionType.User) // + public static final RuntimeOptionKey PrintNMTStatistics = new RuntimeOptionKey<>(false); + @Platforms(Platform.HOSTED_ONLY.class) public static void validateEnableMonitoringFeatures(@SuppressWarnings("unused") OptionKey optionKey) { Set enabledFeatures = getEnabledMonitoringFeatures(); @@ -173,9 +181,6 @@ public static boolean hasNativeMemoryTrackingSupport() { // return hasAllOrKeywordMonitoringSupport(MONITORING_NMT_NAME); } - @Option(help = "Dumps all runtime compiled methods on SIGUSR2.", type = OptionType.User) // - public static final HostedOptionKey DumpRuntimeCompilationOnSignal = new HostedOptionKey<>(false); - static class DeprecatedOptions { @Option(help = "Enables features that allow the VM to be inspected during run time.", type = OptionType.User, // deprecated = true, deprecationMessage = "Please use '--" + ENABLE_MONITORING_OPTION + "'") // diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/headers/LibC.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/headers/LibC.java index a7dd4b55396f..65681458609d 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/headers/LibC.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/headers/LibC.java @@ -35,8 +35,6 @@ import jdk.graal.compiler.api.replacements.Fold; -import jdk.graal.compiler.api.replacements.Fold; - /** Platform-independent LibC support. */ public class LibC { public static final int EXIT_CODE_ABORT = 99; diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/NativeMemoryTracking.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/NativeMemoryTracking.java index 491735517687..9da605cff7e3 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/NativeMemoryTracking.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/NativeMemoryTracking.java @@ -37,8 +37,8 @@ import org.graalvm.word.UnsignedWord; import org.graalvm.word.WordFactory; -import com.oracle.svm.core.SubstrateOptions; import com.oracle.svm.core.Uninterruptible; +import com.oracle.svm.core.VMInspectionOptions; import com.oracle.svm.core.jdk.RuntimeSupport; import com.oracle.svm.core.memory.NativeMemory; import com.oracle.svm.core.util.UnsignedUtils; @@ -147,8 +147,7 @@ public static RuntimeSupport.Hook shutdownHook() { } public void printStatistics() { - // TEMP (chaeubl): disable for now - if (!SubstrateOptions.DiagnosticDetails.getValue().isEmpty()) { + if (VMInspectionOptions.PrintNMTStatistics.getValue()) { System.out.println(); System.out.println("Native memory tracking"); System.out.println(" Total used memory: " + mallocMemorySnapshot.getTotalInfo().getUsed() + " bytes"); diff --git a/substratevm/src/com.oracle.svm.test/src/com/oracle/svm/test/debug/CStructTests.java b/substratevm/src/com.oracle.svm.test/src/com/oracle/svm/test/debug/CStructTests.java index 4bf39fa84729..84e0a7074b94 100644 --- a/substratevm/src/com.oracle.svm.test/src/com/oracle/svm/test/debug/CStructTests.java +++ b/substratevm/src/com.oracle.svm.test/src/com/oracle/svm/test/debug/CStructTests.java @@ -24,6 +24,7 @@ */ package com.oracle.svm.test.debug; +import org.graalvm.nativeimage.UnmanagedMemory; import org.graalvm.nativeimage.c.CContext; import org.graalvm.nativeimage.c.struct.CField; import org.graalvm.nativeimage.c.struct.CFieldAddress; @@ -37,8 +38,6 @@ import org.graalvm.word.SignedWord; import com.oracle.svm.core.NeverInline; -import com.oracle.svm.core.memory.NativeMemory; -import com.oracle.svm.core.nmt.NmtCategory; import jdk.graal.compiler.api.directives.GraalDirectives; @@ -120,7 +119,7 @@ interface Weird extends PointerBase { ]) */ public static void weird() { - Weird wd = NativeMemory.malloc(SizeOf.get(Weird.class), NmtCategory.Other); + Weird wd = UnmanagedMemory.malloc(SizeOf.get(Weird.class)); wd.setf_short((short) 42); wd.setf_int(43); @@ -209,7 +208,7 @@ interface CompositeStruct extends PointerBase { ]) */ public static void composite() { - CompositeStruct cs = NativeMemory.malloc(3 * SizeOf.get(CompositeStruct.class), NmtCategory.Other); + CompositeStruct cs = UnmanagedMemory.malloc(3 * SizeOf.get(CompositeStruct.class)); cs.setC1((byte) 7); cs.setC3(13); cs.setC5((short) 32000); @@ -221,8 +220,8 @@ public static void composite() { } public static void mixedArguments() { - SimpleStruct ss1 = NativeMemory.malloc(SizeOf.get(SimpleStruct.class), NmtCategory.Other); - SimpleStruct2 ss2 = NativeMemory.malloc(SizeOf.get(SimpleStruct2.class), NmtCategory.Other); + SimpleStruct ss1 = UnmanagedMemory.malloc(SizeOf.get(SimpleStruct.class)); + SimpleStruct2 ss2 = UnmanagedMemory.malloc(SizeOf.get(SimpleStruct2.class)); String m1 = "a message in a bottle"; String m2 = "a ship in a bottle"; String m3 = "courage in a bottle"; @@ -252,6 +251,6 @@ public static void main(String[] args) { @NeverInline("Used as a hook to inspect the caller frame in GDB") static void free(PointerBase ptr) { - NativeMemory.free(ptr); + UnmanagedMemory.free(ptr); } } diff --git a/substratevm/src/com.oracle.svm.test/src/com/oracle/svm/test/nmt/NativeMemoryTrackingTests.java b/substratevm/src/com.oracle.svm.test/src/com/oracle/svm/test/nmt/NativeMemoryTrackingTests.java index 3cbacd41288c..7c0a5ce10e37 100644 --- a/substratevm/src/com.oracle.svm.test/src/com/oracle/svm/test/nmt/NativeMemoryTrackingTests.java +++ b/substratevm/src/com.oracle.svm.test/src/com/oracle/svm/test/nmt/NativeMemoryTrackingTests.java @@ -39,7 +39,6 @@ public class NativeMemoryTrackingTests { private static final int ALLOCATION_SIZE = 1024 * 16; - private static final int RESERVE_SIZE = ALLOCATION_SIZE; private static final int REALLOC_SIZE = ALLOCATION_SIZE / 2; @Test From 48e97315db27ce4a174fe2e649ee0e345212dfdd Mon Sep 17 00:00:00 2001 From: Christian Haeubl Date: Tue, 13 Feb 2024 08:21:56 +0100 Subject: [PATCH 7/7] Cleanups. --- .../WindowsSystemPropertiesSupport.java | 2 +- .../oracle/svm/core/VMInspectionOptions.java | 8 ++--- .../core/c/function/IsolateSupportImpl.java | 2 +- .../svm/core/memory/NullableNativeMemory.java | 32 ++++++++++--------- .../svm/core/nmt/NativeMemoryTracking.java | 24 ++++++++------ .../com/oracle/svm/core/nmt/NmtCategory.java | 4 +-- 6 files changed, 38 insertions(+), 34 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core.windows/src/com/oracle/svm/core/windows/WindowsSystemPropertiesSupport.java b/substratevm/src/com.oracle.svm.core.windows/src/com/oracle/svm/core/windows/WindowsSystemPropertiesSupport.java index 5795754098cd..10a35b294b30 100644 --- a/substratevm/src/com.oracle.svm.core.windows/src/com/oracle/svm/core/windows/WindowsSystemPropertiesSupport.java +++ b/substratevm/src/com.oracle.svm.core.windows/src/com/oracle/svm/core/windows/WindowsSystemPropertiesSupport.java @@ -246,7 +246,7 @@ public Pair getOsNameAndVersion() { break; } - VoidPointer versionInfo = NullableNativeMemory.malloc(WordFactory.unsigned(versionSize), NmtCategory.Other); + VoidPointer versionInfo = NullableNativeMemory.malloc(WordFactory.unsigned(versionSize), NmtCategory.Internal); if (versionInfo.isNull()) { break; } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/VMInspectionOptions.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/VMInspectionOptions.java index e5b7ad611a76..55c96fa1234a 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/VMInspectionOptions.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/VMInspectionOptions.java @@ -33,7 +33,6 @@ import org.graalvm.nativeimage.Platform; import org.graalvm.nativeimage.Platform.WINDOWS; import org.graalvm.nativeimage.Platforms; -import org.graalvm.nativeimage.impl.InternalPlatform; import com.oracle.svm.core.heap.dump.HeapDumping; import com.oracle.svm.core.jdk.management.ManagementAgentModule; @@ -81,8 +80,7 @@ public final class VMInspectionOptions { @Option(help = "Dumps all runtime compiled methods on SIGUSR2.", type = OptionType.User) // public static final HostedOptionKey DumpRuntimeCompilationOnSignal = new HostedOptionKey<>(false); - // TEMP (chaeubl): change default to true. - @Option(help = "Print native memory tracking statistics on shutdown.", type = OptionType.User) // + @Option(help = "Print native memory tracking statistics on shutdown if native memory tracking is enabled.", type = OptionType.User) // public static final RuntimeOptionKey PrintNMTStatistics = new RuntimeOptionKey<>(false); @Platforms(Platform.HOSTED_ONLY.class) @@ -176,9 +174,7 @@ public static boolean hasThreadDumpSupport() { @Fold public static boolean hasNativeMemoryTrackingSupport() { - // TEMP (chaeubl): for testing in the CI. - return Platform.includedIn(InternalPlatform.NATIVE_ONLY.class); - // return hasAllOrKeywordMonitoringSupport(MONITORING_NMT_NAME); + return hasAllOrKeywordMonitoringSupport(MONITORING_NMT_NAME); } static class DeprecatedOptions { diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/c/function/IsolateSupportImpl.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/c/function/IsolateSupportImpl.java index 4571f63a383d..a64bc2ac4104 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/c/function/IsolateSupportImpl.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/c/function/IsolateSupportImpl.java @@ -90,7 +90,7 @@ public static IsolateThread createIsolate(CreateIsolateParameters parameters, bo // Internally, we use C-style arguments, i.e., the first argument is reserved for // the name of the binary. We use null when isolates are created manually. argc = isolateArgCount + 1; - argv = NativeMemory.malloc(SizeOf.unsigned(CCharPointerPointer.class).multiply(argc), NmtCategory.Other); + argv = NativeMemory.malloc(SizeOf.unsigned(CCharPointerPointer.class).multiply(argc), NmtCategory.Internal); argv.write(0, WordFactory.nullPointer()); pointerHolders = new CTypeConversion.CCharPointerHolder[isolateArgCount]; diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/memory/NullableNativeMemory.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/memory/NullableNativeMemory.java index 864a2d933e7f..7fa50b7615f6 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/memory/NullableNativeMemory.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/memory/NullableNativeMemory.java @@ -53,9 +53,9 @@ public class NullableNativeMemory { * This method returns a null pointer when allocation fails. */ @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) - public static T malloc(UnsignedWord size, NmtCategory flag) { + public static T malloc(UnsignedWord size, NmtCategory category) { T outerPointer = UntrackedNullableNativeMemory.malloc(getAllocationSize(size)); - return track(outerPointer, size, flag); + return track(outerPointer, size, category); } /** @@ -64,9 +64,9 @@ public static T malloc(UnsignedWord size, NmtCategory fl * This method returns a null pointer when allocation fails. */ @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) - public static T malloc(int size, NmtCategory flag) { + public static T malloc(int size, NmtCategory category) { assert size >= 0; - return malloc(WordFactory.unsigned(size), flag); + return malloc(WordFactory.unsigned(size), category); } /** @@ -75,9 +75,9 @@ public static T malloc(int size, NmtCategory flag) { * This method returns a null pointer when allocation fails. */ @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) - public static T calloc(UnsignedWord size, NmtCategory flag) { + public static T calloc(UnsignedWord size, NmtCategory category) { T outerPointer = UntrackedNullableNativeMemory.calloc(getAllocationSize(size)); - return track(outerPointer, size, flag); + return track(outerPointer, size, category); } /** @@ -86,9 +86,9 @@ public static T calloc(UnsignedWord size, NmtCategory fl * This method returns a null pointer when allocation fails. */ @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) - public static T calloc(int size, NmtCategory flag) { + public static T calloc(int size, NmtCategory category) { assert size >= 0; - return calloc(WordFactory.unsigned(size), flag); + return calloc(WordFactory.unsigned(size), category); } /** @@ -100,9 +100,9 @@ public static T calloc(int size, NmtCategory flag) { */ @SuppressWarnings("unchecked") @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) - public static T realloc(T ptr, UnsignedWord size, NmtCategory flag) { + public static T realloc(T ptr, UnsignedWord size, NmtCategory category) { if (ptr.isNull()) { - return malloc(size, flag); + return malloc(size, category); } else if (!VMInspectionOptions.hasNativeMemoryTrackingSupport()) { return UntrackedNullableNativeMemory.realloc(ptr, getAllocationSize(size)); } @@ -123,7 +123,7 @@ public static T realloc(T ptr, UnsignedWord size, NmtCat /* Only untrack the old block if the allocation was successful. */ NativeMemoryTracking.singleton().untrack(oldSize, oldCategory); - return track(newOuterPointer, size, flag); + return track(newOuterPointer, size, category); } /** @@ -134,9 +134,9 @@ public static T realloc(T ptr, UnsignedWord size, NmtCat * deallocated and remains unchanged. */ @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) - public static T realloc(T ptr, int size, NmtCategory flag) { + public static T realloc(T ptr, int size, NmtCategory category) { assert size >= 0; - return realloc(ptr, WordFactory.unsigned(size), flag); + return realloc(ptr, WordFactory.unsigned(size), category); } /** @@ -167,9 +167,11 @@ private static UnsignedWord getAllocationSize(UnsignedWord size) { @SuppressWarnings("unchecked") @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) - private static T track(T outerPtr, UnsignedWord size, NmtCategory flag) { + private static T track(T outerPtr, UnsignedWord size, NmtCategory category) { if (VMInspectionOptions.hasNativeMemoryTrackingSupport() && outerPtr.isNonNull()) { - return (T) NativeMemoryTracking.singleton().track(outerPtr, size, flag); + T innerPtr = (T) NativeMemoryTracking.singleton().initializeHeader(outerPtr, size, category); + NativeMemoryTracking.singleton().track(innerPtr); + return innerPtr; } return outerPtr; } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/NativeMemoryTracking.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/NativeMemoryTracking.java index 9da605cff7e3..616cbe1944d0 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/NativeMemoryTracking.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/NativeMemoryTracking.java @@ -71,24 +71,25 @@ public static NativeMemoryTracking singleton() { @Fold public static UnsignedWord sizeOfNmtHeader() { /* - * Align the allocation payload to 16 bytes (assuming that the platform-specific malloc - * implementation returns a pointer that is aligned to >= 16 bytes). + * Align the header to 16 bytes to preserve platform-specific malloc alignments up to 16 + * bytes (i.e., the allocation payload is aligned to 16 bytes if the platform-specific + * malloc implementation returns a pointer that is aligned to at least 16 bytes). */ return UnsignedUtils.roundUp(SizeOf.unsigned(NmtMallocHeader.class), ALIGNMENT); } + /** + * Initializes the NMT header and returns a pointer to the allocation payload (i.e., the inner + * pointer). + */ @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) - public Pointer track(PointerBase outerPtr, UnsignedWord size, NmtCategory category) { - /* Initialize the header. */ + @SuppressWarnings("static-method") + public Pointer initializeHeader(PointerBase outerPtr, UnsignedWord size, NmtCategory category) { NmtMallocHeader mallocHeader = (NmtMallocHeader) outerPtr; mallocHeader.setAllocationSize(size); mallocHeader.setCategory(category.ordinal()); assert setMagic(mallocHeader); - - /* Track the memory. */ - Pointer innerPtr = ((Pointer) outerPtr).add(sizeOfNmtHeader()); - track(innerPtr); - return innerPtr; + return getInnerPointer(mallocHeader); } @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) @@ -138,6 +139,11 @@ public static NmtMallocHeader getHeader(PointerBase innerPtr) { return result; } + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + private static Pointer getInnerPointer(NmtMallocHeader mallocHeader) { + return ((Pointer) mallocHeader).add(sizeOfNmtHeader()); + } + public long getUsedMemory(NmtCategory category) { return mallocMemorySnapshot.getInfoByCategory(category).getUsed(); } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/NmtCategory.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/NmtCategory.java index 5cabf99520d3..ff71cdfabaed 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/NmtCategory.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/NmtCategory.java @@ -52,8 +52,8 @@ public enum NmtCategory { /** Memory allocated via Unsafe. */ Unsafe("Unsafe"), - /** Some other VM internal reason - avoid if possible, better to add a new category. */ - Other("Other"); + /** Some other, VM internal reason - avoid if possible, better to add a new category. */ + Internal("Internal"); private final String name;