💚 download arm64 binaries in CI

continuedev · Jan 17, 2024 · 243742c · 243742c
1 parent 51fb2c8
commit 243742c
Show file tree

Hide file tree

Showing 8 changed files with 728 additions and 99 deletions.
diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml
@@ -48,6 +48,11 @@ jobs:
       - name: Check-out repository
         uses: actions/checkout@v4
 
+      - name: Make sure version isn't odd
+        run: |
+          cd extensions/vscode
+          node versionCheck.js
+
       # 2. Install npm dependencies
       - name: Use Node.js 19.0.0
         uses: actions/setup-node@v4

diff --git a/.github/workflows/preview.yaml b/.github/workflows/preview.yaml
@@ -95,62 +95,18 @@ jobs:
           npm ci --legacy-peer-deps
 
       # 2.5. Pre package
-      - name: Make the out directory
+      - name: Make the out/node_modules directory
         run: |
           cd extensions/vscode
           mkdir out
+          cd out
+          mkdir node_modules
 
       - name: Prepare the extension
         run: |
           cd extensions/vscode
           npm run prepackage
 
-      # 2.75. download anything that needs to be built on ARM (non-Windows)
-      - name: Make the out/node_modules directory
-        if: matrix.arch == 'arm64'
-        run: |
-          cd extensions/vscode/out
-          mkdir node_modules
-          cd node_modules
-          mkdir "@esbuild"
-
-      - name: Download esbuild on ARM
-        if: matrix.arch == 'arm64'
-        run: curl -o extensions/vscode/out/node_modules/@esbuild/esbuild.zip https://continue-server-binaries.s3.us-west-1.amazonaws.com/${{ matrix.os }}/esbuild.zip
-
-      - name: Unzip esbuild on ARM
-        if: matrix.arch == 'arm64'
-        run: |
-          cd extensions/vscode/out/node_modules/@esbuild
-          ls
-          unzip esbuild.zip
-          ls
-
-      - name: Remove esbuild.zip on ARM
-        if: matrix.arch == 'arm64'
-        run: rm extensions/vscode/out/node_modules/@esbuild/esbuild.zip
-
-      # 2.875. download anything that needs to be built on ARM (Windows)
-      # - name: Make the out directory
-      #   if: matrix.arch == 'arm64'
-      #   run: |
-      #     cd extensions\vscode
-      #     mkdir out
-
-      # - name: Download node_modules for esbuild on ARM
-      #   if: matrix.arch == 'arm64'
-      #   run: Invoke-WebRequest -Uri https://continue-server-binaries.s3.us-west-1.amazonaws.com/${{ matrix.os }}/node_modules.zip -OutFile extensions\vscode\out\node_modules.zip
-
-      # - name: Unzip node_modules for esbuild on ARM
-      #   if: matrix.arch == 'arm64'
-      #   run: |
-      #     cd extensions\vscode\out
-      #     Expand-Archive -Path node_modules.zip -DestinationPath .
-
-      # - name: Remove node_modules.zip for esbuild on ARM
-      #   if: matrix.arch == 'arm64'
-      #   run: Remove-Item extensions\vscode\out\node_modules.zip
-
       # 3. Run tests for the extension
 
       # - name: Install Xvfb for Linux and run tests
@@ -169,9 +125,12 @@ jobs:
       #   if: matrix.os != 'ubuntu-latest'
 
       # 4. Package the extension
-      - shell: pwsh
+      - name: Set var for environment info
+        shell: pwsh
         run: echo "target=${{ matrix.platform }}-${{ matrix.arch }}" >> $env:GITHUB_ENV
-      - run: cd extensions/vscode && npx vsce package --pre-release --target ${{ env.target }}
+
+      - name: Package the extension
+        run: cd extensions/vscode && npx vsce package --pre-release --target ${{ env.target }}
 
       # 5. Upload the .vsix as an artifact
       - uses: actions/upload-artifact@v2

diff --git a/extensions/vscode/esbuild.js b/extensions/vscode/esbuild.js
@@ -1,9 +1,28 @@
 const esbuild = require("esbuild");
 const ncp = require("ncp").ncp;
 const fs = require("fs");
+const { execSync } = require("child_process");
+const { rimrafSync } = require("rimraf");
+const path = require("path");
+
+function ghAction() {
+  return process.env.target !== undefined;
+}
+
+function isArm() {
+  return (
+    process.env.target === "darwin-arm64" ||
+    process.env.target === "linux-arm64" ||
+    process.env.target === "win-arm64"
+  );
+}
+
+function isWin() {
+  return process.env.target?.startsWith("win");
+}
 
 (async () => {
-  console.log("Bundling with esbuild...");
+  console.log("Bundling with esbuild for target ", process.env.target);
 
   // Bundles the extension into one file
   await esbuild.build({
@@ -25,32 +44,68 @@ const fs = require("fs");
     define: { "import.meta.url": "importMetaUrl" },
   });
 
-  fs.mkdirSync("out/node_modules", { recursive: true });
+  // GitHub Actions doesn't support ARM, so we need to download pre-saved binaries
+  if (ghAction() && isArm()) {
+    // Download and unzip esbuild
+    console.log("Downloading pre-built esbuild binary");
+    rimrafSync("node_modules/@esbuild", {
+      preserveRoot: true,
+    });
+    execSync(
+      `curl -o node_modules/@esbuild/esbuild.zip https://continue-server-binaries.s3.us-west-1.amazonaws.com/${process.env.target}/esbuild.zip`
+    );
+    execSync(`cd node_modules/@esbuild && ls && unzip esbuild.zip && ls`);
+    fs.unlinkSync("node_modules/@esbuild/esbuild.zip");
+
+    // Neither lancedb nor sqlite3 have pre-built windows arm64 binaries
+    if (!isWin()) {
+      // lancedb binary
+      console.log("Downloading pre-built lancedb binary");
+      rimrafSync("node_modules/@lancedb");
+      const packageToInstall = {
+        "darwin-arm64": "@lancedb/vectordb-darwin-arm64",
+        "linux-arm64": "@lancedb/vectordb-linux-arm64-gnu",
+      }[process.env.target];
+      execSync(`npm install ${packageToInstall}`);
+    }
+  }
+
+  if (ghAction()) {
+    // sqlite3
+    if (isArm() && !isWin()) {
+      // Replace the installed with pre-built
+      console.log("Downloading pre-built sqlite3 binary");
+      rimrafSync("../../core/node_modules/sqlite3/build");
+      const downloadUrl = {
+        "darwin-arm64":
+          "https://github.com/TryGhost/node-sqlite3/releases/download/v5.1.7/sqlite3-v5.1.7-napi-v6-darwin-arm64.tar.gz",
+        "linux-arm64":
+          "https://github.com/TryGhost/node-sqlite3/releases/download/v5.1.7/sqlite3-v5.1.7-napi-v3-linux-arm64.tar.gz",
+      };
+      execSync(
+        `curl -o ../../core/node_modules/sqlite3/build.zip ${downloadUrl}`
+      );
+      execSync("cd ../../core/node_modules/sqlite3 && unzip build.zip");
+      fs.unlinkSync("../../core/node_modules/sqlite3/build.zip");
+    }
 
-  const NODE_MODULES_TO_COPY = ["esbuild", "@lancedb"];
+    ncp(
+      path.join(__dirname, "../../core/node_modules/sqlite3/build"),
+      path.join(__dirname, "out/build"),
+      (error) => {
+        if (error) console.warn("Error copying sqlite3 files", error);
+      }
+    );
+  }
+
+  // Copy node_modules for pre-built binaries
+  const NODE_MODULES_TO_COPY = ["esbuild", "@esbuild", "@lancedb"];
+  fs.mkdirSync("out/node_modules", { recursive: true });
   NODE_MODULES_TO_COPY.forEach((mod) => {
     ncp.ncp(`node_modules/${mod}`, `out/node_modules/${mod}`, function (err) {
       if (err) {
         return console.error(err);
       }
     });
   });
-
-  // Return instead of copying if on ARM
-  // This is an env var created in the GH Action
-  // We will instead download the prebuilt binaries
-  if (
-    process.env.target === "darwin-arm64" ||
-    process.env.target === "linux-arm64" ||
-    process.env.target === "win-arm64"
-  ) {
-    console.log("Skipping copying binaries");
-    return;
-  }
-
-  ncp.ncp("node_modules/@esbuild", "out/node_modules/@esbuild", function (err) {
-    if (err) {
-      return console.error(err);
-    }
-  });
 })();