Merge pull request #5365 from rjbou/gha

GHA: port to new infra, upgrade actions version

.github/scripts/cygwin.cmd

@@ -15,19 +15,17 @@
 ::
 :: cygwin.cmd distro cache-directory {create|host}
 ::
-:: where distro is i686-pc-cygwin or x86_64-pc-cygwin and rebuilds the cache
+:: where distro is x86_64-pc-cygwin and rebuilds the cache
 ::
 :: Environment variables:
 ::   CYGWIN_ROOT        - Cygwin installation root directory
 ::   CYGWIN_MIRROR      - Package repository mirror
 
 set CYGWIN_CACHE_DIR=%2
 set CYGWIN_DISTRO=%1
-if "%CYGWIN_DISTRO%" neq "i686-pc-cygwin" (
-  if "%CYGWIN_DISTRO%" neq "x86_64-pc-cygwin" (
-    echo Invalid Cygwin distro: %1
-    exit /b 2
-  )
+if "%CYGWIN_DISTRO%" neq "x86_64-pc-cygwin" (
+  echo Invalid Cygwin distro: %1
+  exit /b 2
 )
 
 if "%3" equ "create" goto SetupCygwin
@@ -53,7 +51,6 @@ if not exist %CYGWIN_CACHE_DIR%\%CYGWIN_DISTRO%\cache.tar (
 set Path=C:\Program Files\Mercurial;C:\Program Files\Git\cmd;%CYGWIN_ROOT%\bin;C:\Windows\system32;C:\Windows;C:\Windows\System32\Wbem;C:\Windows\System32\WindowsPowerShell\v1.0\;C:\Windows\System32\OpenSSH\
 if "%3" equ "i686-w64-mingw32" set Path=%CYGWIN_ROOT%\usr\%3\sys-root\mingw\bin;%Path%
 if "%3" equ "x86_64-w64-mingw32" set Path=%CYGWIN_ROOT%\usr\%3\sys-root\mingw\bin;%Path%
-if "%3" equ "i686-pc-cygwin" set Path=%CYGWIN_ROOT%\bin;%Path%
 if "%3" equ "x86_64-pc-cygwin" set Path=%CYGWIN_ROOT%\bin;%Path%
 
 ::echo %CYGWIN_ROOT%\bin>> %GITHUB_PATH%

.github/scripts/main/create-ocaml-cache.sh

@@ -0,0 +1,37 @@
+#!/bin/bash -xue
+
+OCAML_BRANCH="$1"
+PREFIX="$2"
+EXE="$3"
+OCAML_LOCAL="$4"
+PLATFORM="$5"
+
+if [[ $OCAML_BRANCH -gt 407 ]]; then
+  if [[ -n $GITHUB_BASE_REF ]]; then
+    git tag combak
+    git fetch origin $GITHUB_BASE_REF
+    git checkout origin/$GITHUB_BASE_REF
+  fi
+  make -C src_ext dune-local.stamp
+  cd src_ext/dune-local
+  ocaml bootstrap.ml
+  cp dune.exe "$PREFIX/bin/dune$EXE"
+  cd ../..
+
+  ./configure
+  make
+  cp -a _build "$OCAML_LOCAL/"
+  rm -f "$OCAML_LOCAL/_build/log"
+  mv "$OCAML_LOCAL/_build/default/src_ext" "$OCAML_LOCAL/_build/"
+  rm -rf "$OCAML_LOCAL/_build/default"/* "$OCAML_LOCAL/_build/install"
+  mv "$OCAML_LOCAL/_build/src_ext" "$OCAML_LOCAL/_build/default/" 
+  git clean -dfX
+  if [[ -n $GITHUB_BASE_REF ]]; then
+    git checkout combak
+  fi
+fi
+
+# The Windows BSD tar can't cope with symlinks, so we pre-tar the archive and cache that!
+if [[ $PLATFORM = 'Windows' ]]; then
+  tar -C "$OCAML_LOCAL" -pcf "$OCAML_LOCAL.tar" .
+fi

.github/scripts/main/ocaml-cache.sh

@@ -51,7 +51,21 @@ fi
 
 tar -xzf "ocaml-$OCAML_VERSION.tar.gz"
 
+case "${OCAML_VERSION%.*}" in
+  4.08) PATCHES='e322556b0a9097a2eff2117476193b773e1b947f 17df117b4939486d3285031900587afce5262c8c';;
+  4.09) PATCHES='8eed2e441222588dc385a98ae8bd6f5820eb0223';;
+  4.10) PATCHES='4b4c643d1d5d28738f6d900cd902851ed9dc5364';;
+  4.11) PATCHES='dd28ac0cf4365bd0ea1bcc374cbc5e95a6f39bea';;
+  4.12) PATCHES='1eeb0e7fe595f5f9e1ea1edbdf785ff3b49feeeb';;
+  *) PATCHES='';;
+esac
+
 cd "ocaml-$OCAML_VERSION"
+for sha in $PATCHES; do
+  curl -sL "https://github.com/ocaml/ocaml/commit/$sha.patch" -o "../$sha.patch"
+  patch -p1 -i "../$sha.patch"
+done
+
 if [[ $PLATFORM = 'Windows' ]] ; then
   tar -xzf ../$FLEXDLL_VERSION.tar.gz
   rm -rf flexdll
@@ -130,32 +144,8 @@ EOF
   chmod +x "$OCAML_LOCAL/bin/ocamldoc"
 fi
 
-if [[ $OCAML_BRANCH -gt 407 ]]; then
-  if [[ -n $GITHUB_BASE_REF ]]; then
-    git tag combak
-    git fetch origin $GITHUB_BASE_REF
-    git checkout origin/$GITHUB_BASE_REF
-  fi
-  make -C src_ext dune-local.stamp
-  cd src_ext/dune-local
-  ocaml bootstrap.ml
-  cp dune.exe "$PREFIX/bin/dune$EXE"
-  cd ../..
-
-  ./configure
-  make
-  cp -a _build "$OCAML_LOCAL/"
-  rm -f "$OCAML_LOCAL/_build/log"
-  mv "$OCAML_LOCAL/_build/default/src_ext" "$OCAML_LOCAL/_build/"
-  rm -rf "$OCAML_LOCAL/_build/default"/* "$OCAML_LOCAL/_build/install"
-  mv "$OCAML_LOCAL/_build/src_ext" "$OCAML_LOCAL/_build/default/" 
-  git clean -dfX
-  if [[ -n $GITHUB_BASE_REF ]]; then
-    git checkout combak
-  fi
-fi
-
-# The Windows BSD tar can't cope with symlinks, so we pre-tar the archive and cache that!
-if [[ $PLATFORM = 'Windows' ]]; then
-  tar -C "$OCAML_LOCAL" -pcf "$OCAML_LOCAL.tar" .
-fi
+# Hand-over control to a separate script in case the branch being tested
+# updates this script, which will fail on Windows (since the script is "open"
+# and can't be overwritten)
+cp -pf .github/scripts/main/create-ocaml-cache.sh ../create-ocaml-cache.sh
+exec ../create-ocaml-cache.sh "$OCAML_BRANCH" "$PREFIX" "$EXE" "$OCAML_LOCAL" "$PLATFORM"

.github/workflows/ci.ml

@@ -107,7 +107,7 @@ type cache = {
 type _ cache_name =
 | Archives : ((cache -> 'a) -> 'a) cache_name
 | Secondary : ((cache -> 'a) -> 'a) cache_name
-| Cygwin : ((cache -> 'a) -> [ `x86 | `x64 ] -> 'a) cache_name
+| Cygwin : ((cache -> 'a) -> 'a) cache_name
 | OCaml : ((cache -> 'a) -> _ platform -> string -> string -> 'a) cache_name
 | OpamBS : ((cache -> 'a) -> string -> string -> 'a) cache_name
 | OpamRoot : ((cache -> 'a) -> string -> 'a) cache_name
@@ -137,15 +137,14 @@ let get_cache_cont : type s . s cache_name -> s = function
          build = ["bash -exu .github/scripts/main/legacy-cache.sh"];
          build_shell = None}
   | Cygwin ->
-      fun f arch ->
-        let (arch_name, key_arch) = match arch with `x86 -> ("i686", "32") | `x64 -> ("x86_64", "64") in f
-        {name = (match arch with `x86 -> "Cygwin32" | `x64 -> "Cygwin64");
-         key = Printf.sprintf "cygwin%s-${{ %s.outputs.cygwin }}" key_arch;
-         id = "cygwin" ^ key_arch;
+      fun f -> f
+        {name = "Cygwin64";
+         key = Printf.sprintf "cygwin64-${{ %s.outputs.cygwin }}";
+         id = "cygwin64";
          force_gzip = true;
-         paths = [Printf.sprintf "%s\\%s-pc-cygwin" cygwin_cache_directory arch_name];
+         paths = [Printf.sprintf "%s\\x86_64-pc-cygwin" cygwin_cache_directory];
          always_build = false;
-         build = [Printf.sprintf {|.github\scripts\cygwin.cmd %s-pc-cygwin %s create|} arch_name cygwin_cache_directory];
+         build = [Printf.sprintf {|.github\scripts\cygwin.cmd x86_64-pc-cygwin %s create|} cygwin_cache_directory];
          build_shell = Some "cmd"}
   | OCaml ->
       fun f (type a) (platform : a platform) version host ->
@@ -202,7 +201,7 @@ let cache ?cond ?(key_prefix="needs.Analyse") ?(check_only=false) name =
       if cache.force_gzip || check_only then
         "ocaml-opam/cache@opam"
       else
-        "actions/cache@v2"
+        "actions/cache@v3"
     in
     let withs =
       if cache.force_gzip then
@@ -236,6 +235,7 @@ let unpack_cygwin ?cond build host =
   run ?cond ~shell:"cmd" "Unpack Cygwin" [Printf.sprintf {|.github\scripts\cygwin.cmd %s %s %s|} build cygwin_cache_directory host]
 
 let install_sys_packages packages ~descr ?cond platforms =
+  let platforms = List.map os_of_platform platforms in
   let packages = String.concat " " packages in
   let linux_command = "sudo apt install " ^ packages in
   let macos_command = "brew install " ^ packages in
@@ -271,30 +271,31 @@ let install_sys_opam ?cond = install_sys_packages ["opam"] ~descr:"Install syste
 let install_sys_dune ?cond = install_sys_packages ["dune"; "ocaml"] ~descr:"Install system's dune and ocaml packages" ?cond
 
 let analyse_job ~oc ~workflow ~platforms ~keys f =
+  let oses = List.map os_of_platform platforms in
   let outputs =
     let f (key, _) = (key, Printf.sprintf "${{ steps.keys.outputs.%s }}" key) in
     List.map f keys
   in
   let keys =
     let set_key (name, value) =
       [Printf.sprintf "echo %s=%s" name value;
-       Printf.sprintf "echo ::set-output name=%s::%s" name value]
+       Printf.sprintf "echo %s=%s >> $GITHUB_OUTPUT" name value]
     in
     List.flatten (List.map set_key keys)
   in
   let only_with platform step =
-    if List.mem platform platforms then
+    if List.mem platform oses then
       step
     else
       skip_step
   in
   let linux_guard =
-    match platforms with
+    match oses with
     | [Linux] -> None
     | _ -> Some (Predicate(true, Runner Linux))
   in
   let not_windows_guard =
-    if List.mem Windows platforms then
+    if List.mem Windows oses then
       Some (Predicate(false, Runner Windows))
     else
       None
@@ -311,20 +312,15 @@ let analyse_job ~oc ~workflow ~platforms ~keys f =
     ++ end_job f
 
 let cygwin_job ~analyse_job ~oc ~workflow f =
-  let cygwin32 = get_cache Cygwin `x86 in
-  let cygwin64 = get_cache Cygwin `x64 in
-  let either_missed =
-    Or(Predicate(true, CacheMiss cygwin32.id), Predicate(true, CacheMiss cygwin64.id))
-  in
+  let cygwin64 = get_cache Cygwin in
   job ~oc ~workflow ~runs_on:(Runner [Windows]) ~needs:[analyse_job] "Cygwin"
-    ++ cache ~check_only:true Cygwin `x86 
-    ++ cache ~check_only:true Cygwin `x64
-    ++ checkout ~cond:either_missed ()
-    ++ build_cache Cygwin `x86
-    ++ build_cache Cygwin `x64
+    ++ cache ~check_only:true Cygwin
+    ++ checkout ~cond:(Predicate(true, CacheMiss cygwin64.id)) ()
+    ++ build_cache Cygwin
     ++ end_job f
 
-let main_build_job ~analyse_job ~cygwin_job ?section platform start_version ~oc ~workflow f =
+let main_build_job ~analyse_job ~cygwin_job ?section runner start_version ~oc ~workflow f =
+  let platform = os_of_platform runner in
   let only_on target = only_on platform target in
   let not_on target = not_on platform target in
   let shell =
@@ -338,30 +334,19 @@ let main_build_job ~analyse_job ~cygwin_job ?section platform start_version ~oc
   let (_fail_fast, matrix, _) = platform_ocaml_matrix ~fail_fast:true start_version in
   let (matrix, includes) =
     if platform = Windows then
-      let includes =
-        (* Annoyingly, it does not appear to be possible to exclude or include by pattern, i.e. to
-           have
-           include:
-              - host: i686-pc-cygwin
-                build: i686-pc-cygwin
-            and have GHA take the cross-product of that with ocamlv from the main matrix
-            So we have to duplicate the ocamlv below... *)
-        List.map (fun v -> [("ocamlv", v); ("host", "i686-pc-cygwin"); ("build", "i686-pc-cygwin")]) (snd (List.hd matrix))
-      in
       (("host", ["x86_64-pc-cygwin"; "i686-w64-mingw32"; "x86_64-w64-mingw32"; "i686-pc-windows"; "x86_64-pc-windows"]) ::
        ("build", ["x86_64-pc-cygwin"]) ::
-        matrix, includes)
+       matrix, [])
     else
       (matrix, []) in
   let matrix = ((platform <> Windows), matrix, includes) in
   let needs = if platform = Windows then [analyse_job; cygwin_job] else [analyse_job] in
   let host = host_of_platform platform in
-  job ~oc ~workflow ~runs_on:(Runner [platform]) ?shell ?section ~needs ~matrix ("Build-" ^ name_of_platform platform)
+  job ~oc ~workflow ~runs_on:(Runner [runner]) ?shell ?section ~needs ~matrix ("Build-" ^ name_of_platform platform)
     ++ only_on Linux (run "Install bubblewrap" ["sudo apt install bubblewrap"])
     ++ only_on Windows (git_lf_checkouts ~cond:(Predicate(true, EndsWith("matrix.host", "-pc-cygwin"))) ~shell:"cmd" ~title:"Configure LF checkout for Cygwin" ())
     ++ checkout ()
-    ++ only_on Windows (cache ~cond:(Predicate(true, Compare("matrix.build", "i686-pc-cygwin"))) Cygwin `x86)
-    ++ only_on Windows (cache ~cond:(Predicate(true, Compare("matrix.build", "x86_64-pc-cygwin"))) Cygwin `x64)
+    ++ only_on Windows (cache ~cond:(Predicate(true, Compare("matrix.build", "x86_64-pc-cygwin"))) Cygwin)
     ++ cache Archives
     ++ cache OCaml platform "${{ matrix.ocamlv }}" host
     ++ only_on Windows (unpack_cygwin "${{ matrix.build }}" "${{ matrix.host }}")
@@ -403,7 +388,8 @@ let legacy_build_job ~analyse_job ~build_linux_job ~build_windows_job ~build_mac
     ++ run "Build" ["bash -exu .github/scripts/main/legacy.sh"]
     ++ end_job f
 
-let main_test_job ~analyse_job ~build_linux_job ~build_windows_job:_ ~build_macOS_job:_ ?section platform ~oc ~workflow f =
+let main_test_job ~analyse_job ~build_linux_job ~build_windows_job:_ ~build_macOS_job:_ ?section runner ~oc ~workflow f =
+  let platform = os_of_platform runner in
   let _ = assert (platform <> Windows) in
   let only_on target = only_on platform target in
   let needs =
@@ -415,7 +401,7 @@ let main_test_job ~analyse_job ~build_linux_job ~build_windows_job:_ ~build_macO
   let matrix = platform_ocaml_matrix ~fail_fast:false latest_ocaml in
   let host = host_of_platform platform in
   let ocamlv = "${{ matrix.ocamlv }}" in
-  job ~oc ~workflow ?section ~runs_on:(Runner [platform]) ~env:[("OPAM_TEST", "1")] ~matrix ~needs ("Test-" ^ name_of_platform platform)
+  job ~oc ~workflow ?section ~runs_on:(Runner [runner]) ~env:[("OPAM_TEST", "1")] ~matrix ~needs ("Test-" ^ name_of_platform platform)
     ++ checkout ()
     ++ only_on Linux (run "Install bubblewrap" ["sudo apt install bubblewrap"])
     ++ cache Archives
@@ -428,12 +414,13 @@ let main_test_job ~analyse_job ~build_linux_job ~build_windows_job:_ ~build_macO
     ++ run "Test (opam-rt)" ["bash -exu .github/scripts/main/opam-rt.sh"]
     ++ end_job f
 
-let cold_job ~analyse_job ~build_linux_job ~build_windows_job ~build_macOS_job ?section platform ~oc ~workflow f =
+let cold_job ~analyse_job ~build_linux_job ~build_windows_job ~build_macOS_job ?section runner ~oc ~workflow f =
+  let platform = os_of_platform runner in
   (* TODO Windows steps not all here *)
   let host = host_of_platform platform in
   let only_on target = only_on platform target in
   let needs = [analyse_job; (match platform with Linux -> build_linux_job | Windows -> build_windows_job | MacOS -> build_macOS_job)] in
-  job ~oc ~workflow ?section ~runs_on:(Runner [platform]) ~env:[("OPAM_COLD", "1")] ~needs ("Cold-" ^ name_of_platform platform)
+  job ~oc ~workflow ?section ~runs_on:(Runner [runner]) ~env:[("OPAM_COLD", "1")] ~needs ("Cold-" ^ name_of_platform platform)
     ++ only_on Linux (run "Install bubblewrap" ["sudo apt install bubblewrap"])
     ++ checkout ()
     ++ cache Archives
@@ -444,7 +431,8 @@ let cold_job ~analyse_job ~build_linux_job ~build_windows_job ~build_macOS_job ?
     ++ run "Test (basic)" ["bash -exu .github/scripts/main/test.sh"]
     ++ end_job f
 
-let solvers_job ~analyse_job ~build_linux_job ~build_windows_job ~build_macOS_job ?section platform ~oc ~workflow f =
+let solvers_job ~analyse_job ~build_linux_job ~build_windows_job ~build_macOS_job ?section runner ~oc ~workflow f =
+  let platform = os_of_platform runner in
   (* TODO Windows steps not all here *)
   let host = host_of_platform platform in
   let only_on target = only_on platform target in
@@ -455,7 +443,7 @@ let solvers_job ~analyse_job ~build_linux_job ~build_windows_job ~build_macOS_jo
     (fail_fast, ("solver", ["z3"; "0install"])::matrix, [])
   in
   let ocamlv = "${{ matrix.ocamlv }}" in
-  job ~oc ~workflow ?section ~runs_on:(Runner [platform]) ~env ~needs ~matrix ("Solvers-" ^ name_of_platform platform)
+  job ~oc ~workflow ?section ~runs_on:(Runner [runner]) ~env ~needs ~matrix ("Solvers-" ^ name_of_platform platform)
     ++ only_on Linux (run "Install bubblewrap" ["sudo apt install bubblewrap"])
     ++ checkout ()
     ++ cache Archives
@@ -466,14 +454,15 @@ let solvers_job ~analyse_job ~build_linux_job ~build_windows_job ~build_macOS_jo
     ++ run "Compile" ["bash -exu .github/scripts/main/solvers.sh"]
     ++ end_job f
 
-let upgrade_job ~analyse_job ~build_linux_job ~build_windows_job ~build_macOS_job ?section platform ~oc ~workflow f =
+let upgrade_job ~analyse_job ~build_linux_job ~build_windows_job ~build_macOS_job ?section runner ~oc ~workflow f =
+  let platform = os_of_platform runner in
   let _ = assert (platform <> Windows) in (* No opam 1.x for Windows *)
   let host = host_of_platform platform in
   let only_on target = only_on platform target in
   let needs = [analyse_job; (match platform with Linux -> build_linux_job | Windows -> build_windows_job | MacOS -> build_macOS_job)] in
   let matrix = platform_ocaml_matrix ~fail_fast:false latest_ocaml in
   let ocamlv = "${{ matrix.ocamlv }}" in
-  job ~oc ~workflow ?section ~runs_on:(Runner [platform]) ~needs ~matrix ("Upgrade-" ^ name_of_platform platform)
+  job ~oc ~workflow ?section ~runs_on:(Runner [runner]) ~needs ~matrix ("Upgrade-" ^ name_of_platform platform)
     ++ only_on Linux (run "Install bubblewrap" ["sudo apt install bubblewrap"])
     ++ checkout ()
     ++ cache Opam12Root
@@ -488,7 +477,7 @@ let hygiene_job (type a) ~analyse_job (platform : a platform) ~oc ~workflow f =
     ++ install_sys_dune [os_of_platform platform]
     ++ checkout ()
     ++ cache Archives
-    ++ uses "Get changed files" ~id:"files" ~continue_on_error:true (* see https://github.com/jitterbit/get-changed-files/issues/19 *) "jitterbit/get-changed-files@v1"
+    ++ uses "Get changed files" ~id:"files" (* ~continue_on_error:true see https://github.com/jitterbit/get-changed-files/issues/19 *) "Ana06/get-changed-files@v2.2.0" (* see https://github.com/jitterbit/get-changed-files/issues/55 ; Ana06'fork contains #19 and #55 fixes *) 
     ++ run "Changed files list" [
          "for changed_file in ${{ steps.files.outputs.modified }}; do";
          "  echo \"M  ${changed_file}.\"";
@@ -525,7 +514,7 @@ let main oc : unit =
   let keys = [
     ("archives", "archives-1-${{ hashFiles('src_ext/Makefile.sources', 'src_ext/Makefile', '.github/scripts/common/preamble.sh', '.github/scripts/main/preamble.sh', '.github/scripts/main/archives-cache.sh') }}-${{ env.OPAM_REPO_SHA }}");
     ("ocaml-secondary-compiler", "legacy-${{ env.OPAM_REPO_SHA }}");
-    ("ocaml-cache", "${{ hashFiles('.github/scripts/main/ocaml-cache.sh', '.github/scripts/main/preamble.sh') }}");
+    ("ocaml-cache", "${{ hashFiles('.github/scripts/main/ocaml-cache.sh', '.github/scripts/main/preamble.sh', '.github/scripts/main/create-ocaml-cache.sh') }}");
     ("cygwin", "${{ hashFiles('.github/scripts/cygwin.cmd') }}-${{ env.CYGWIN_EPOCH }}");
     ("opam-bs-cache", "${{ hashFiles('.github/scripts/main/opam-bs-cache.sh', '*.opam', '.github/scripts/main/preamble.sh') }}");
   ] in