Integration/3.0.0.2 (#34)

* automatically update pools

* persist system memory size and autoconfig bitcoin maxmempool accordingly

* bump versions and update release notes

* cleanup

Dockerfile

@@ -9,6 +9,7 @@ ENV MEMPOOL_INDEXING_BLOCKS_AMOUNT="52560"
 ENV MEMPOOL_STDOUT_LOG_MIN_PRIORITY="info"
 ENV LIGHTNING_STATS_REFRESH_INTERVAL=3600
 ENV LIGHTNING_GRAPH_REFRESH_INTERVAL=3600
+ENV MEMPOOL_AUTOMATIC_POOLS_UPDATE=true
 
 USER root
 # arm64 or amd64

docker_entrypoint.sh

@@ -8,6 +8,18 @@ _term() {
   kill -TERM "$frontend_process" 2>/dev/null
 }
 
+# Record system memory info to read for bitcoin auto config
+SYSTEM_MEM_INFO=$(awk '/MemTotal/{print $(NF-1)}' /proc/meminfo)
+SYSTEM_MEM_FILE=/root/start9/system_mem_info
+
+if [ -f "$SYSTEM_MEM_FILE" ]
+then
+	echo "$SYSTEM_MEM_INFO" > "$SYSTEM_MEM_FILE"
+else
+	touch /root/start9/system_mem_info
+	echo "$SYSTEM_MEM_INFO" > "$SYSTEM_MEM_FILE"
+fi
+
 # FRONTEND SETUP
 
 LIGHTNING_DETECTED_PORT=9735
@@ -40,6 +52,9 @@ sed -i "s/CORE_RPC_HOST:=127.0.0.1/CORE_RPC_HOST:=$bitcoind_host/" start.sh
 sed -i "s/CORE_RPC_USERNAME:=mempool/CORE_RPC_USERNAME:=$bitcoind_user/" start.sh
 sed -i "s/CORE_RPC_PASSWORD:=mempool/CORE_RPC_PASSWORD:=$bitcoind_pass/" start.sh
 
+# adjust heap size
+sed -i "s/node \/backend\/package\/index.js/node --max-old-space-size=16384 \/backend\/package\/index.js/" start.sh
+
 # Configure mempool to set lightning to true if lightning is enabled
 if [ "$(yq e ".lightning.type" /root/start9/config.yaml)" = "none" ]; then
 	sed -i 's/LIGHTNING_ENABLED:=true/LIGHTNING_ENABLED:=false/' start.sh

manifest.yaml

@@ -1,8 +1,10 @@
 id: mempool
 title: Mempool
-version: 3.0.0.1
+version: 3.0.0.2
 release-notes: |
-  * Exciting new version which adds the Mempool Accelerator among other improvements and bugfixes. Check out the full [release notes](https://github.com/mempool/mempool/releases/tag/v3.0.0).
+  * Sets bounds on RAM used for the mempool
+  * Update the latest pool data from the Mempool repository on startup
+  * Adjust JS heap limit
 license: AGPL
 wrapper-repo: "https://github.com/Start9Labs/mempool-wrapper"
 upstream-repo: "https://github.com/mempool/mempool"

scripts/procedures/dependencies.ts

@@ -1,19 +1,35 @@
 import { matches, types as T } from "../deps.ts";
 
-const { shape, string, boolean } = matches;
+const { shape, string, boolean, number } = matches;
 
 const matchBitcoindConfig = shape({
   advanced: shape({
     pruning: shape({
       mode: string,
     }),
+    mempool: shape({
+      maxmempool: number
+    })
   }),
   rpc: shape({
     enable: boolean,
   }),
   txindex: boolean,
 });
 
+async function getSystemMemoryLimit(effects: T.Effects) {
+  try {
+    const memdata = await effects.readFile({ volumeId: 'main', path: 'start9/system_mem_info'})
+    // convert kb to mb
+    const memMB = parseInt(memdata) / 1000
+    return Math.round(memMB / 6)
+  } catch (e) {
+    effects.info(e)
+    // default recommended is 300MB
+    return 300
+  }
+}
+
 export const dependencies: T.ExpectedExports.dependencies = {
   bitcoind: {
     // deno-lint-ignore require-await
@@ -32,6 +48,12 @@ export const dependencies: T.ExpectedExports.dependencies = {
             "Pruning must be disabled to use Bitcoin Core.",
         };
       }
+      const limit = await getSystemMemoryLimit(effects)
+      if (limit > 300 && config.advanced.mempool.maxmempool >= limit) {
+        return {
+          error: "In order to safely run Mempool, Bitcoin Core's \"maxmempool\" size cannot exceed 1/6 of the system RAM"
+        }
+      }
       return { result: null };
     },
     // deno-lint-ignore require-await
@@ -41,7 +63,11 @@ export const dependencies: T.ExpectedExports.dependencies = {
       config.rpc.enable = true;
       config.txindex = true;
       config.advanced.pruning.mode = "disabled";
+      const limit = await getSystemMemoryLimit(effects)
+      if (limit > 300 && config.advanced.mempool.maxmempool >= limit) {
+        config.advanced.mempool.maxmempool = limit
+      }
       return { result: config };
     },
   }
-};
+};