Merge pull request #6303 from kit-ty-kate/swhid-wget2

Avoid issues when using wget2 where the requested url might return an html page instead of the expected content
ocaml · Nov 22, 2024 · 94134cc · 94134cc
2 parents 1a347b4 + f2cc2e0
commit 94134cc
Show file tree

Hide file tree

Showing 3 changed files with 11 additions and 3 deletions.
diff --git a/master_changes.md b/master_changes.md
@@ -118,6 +118,7 @@ users)
   * Make `curl` the default download tool instead of `wget` on macOS [#6304 @kit-ty-kate]
   * download tool: Use fetch on DragonFlyBSD and ftp on NetBSD [#6305 @kit-ty-kate]
   * Prefer curl over any other download tools on every systems, if available [#6305 @kit-ty-kate]
+  * Avoid issues when using wget2 where the requested url might return an html page instead of the expected content [#6303 @kit-ty-kate]
 
 ## Internal: Windows
 

diff --git a/src/repository/opamDownload.ml b/src/repository/opamDownload.ml
@@ -44,6 +44,13 @@ let curl_args =
       main_args
 
 let wget_args = [
+  (* wget2 by default has
+     Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8
+     which makes some servers return an html page instead of the expected
+     content. This is the case for the Software Heritage REST API.
+     Using "Accept: */*" makes it on par with curl
+  *)
+  CString "--header=Accept: */*", None;
   CString "-t", None; CIdent "retry", None;
   CString "-O", None; CIdent "out", None;
   CString "-U", None; user_agent, None;

diff --git a/tests/reftests/download.test b/tests/reftests/download.test
@@ -33,7 +33,7 @@ The following actions will be performed:
 
 <><> Processing actions <><><><><><><><><><><><><><><><><><><><><><><><><><><><>
 Processing  1/1: [foo.1: http]
-+ wget "-t" "3" "-O" "${OPAMTMP}/v1.0.0.tar.gz.part" "-U" "opam/current" "--" "https://github.com/UnixJunkie/get_line/archive/v1.0.0.tar.gz"
++ wget "--header=Accept: */*" "-t" "3" "-O" "${OPAMTMP}/v1.0.0.tar.gz.part" "-U" "opam/current" "--" "https://github.com/UnixJunkie/get_line/archive/v1.0.0.tar.gz"
 Processing  1/1: [foo.1: extract]
 + tar "xfz" "${OPAMTMP}/v1.0.0.tar.gz" "-C" "${OPAMTMP}"
 Done.
@@ -147,7 +147,7 @@ The following actions will be performed:
 
 <><> Processing actions <><><><><><><><><><><><><><><><><><><><><><><><><><><><>
 Processing  1/1: [foo.1: http]
-+ wget "-t" "3" "-O" "${OPAMTMP}/v1.0.0.tar.gz.part" "-U" "opam/current" "--" "https://github.com/UnixJunkie/get_line/archive/v1.0.0.tar.gz"
++ wget "--header=Accept: */*" "-t" "3" "-O" "${OPAMTMP}/v1.0.0.tar.gz.part" "-U" "opam/current" "--" "https://github.com/UnixJunkie/get_line/archive/v1.0.0.tar.gz"
 Processing  1/1: [foo.1: extract]
 + tar "xfz" "${OPAMTMP}/v1.0.0.tar.gz" "-C" "${OPAMTMP}"
 Done.
@@ -260,7 +260,7 @@ The following actions will be performed:
 SYSTEM                          rmdir ${BASEDIR}/OPAM/download/.opam-switch/sources/baz.1
 SYSTEM                          mkdir ${OPAMTMP}
 Processing  1/1: [baz.1: http]
-+ wget "-t" "3" "-O" "${OPAMTMP}/v1.0.0.tar.gz.part" "-U" "opam/current" "--" "https://github.com/UnixJunkie/get_line/archive/v1.0.0.tar.gz"
++ wget "--header=Accept: */*" "-t" "3" "-O" "${OPAMTMP}/v1.0.0.tar.gz.part" "-U" "opam/current" "--" "https://github.com/UnixJunkie/get_line/archive/v1.0.0.tar.gz"
 SYSTEM                          mv ${OPAMTMP}/v1.0.0.tar.gz.part -> ${OPAMTMP}/v1.0.0.tar.gz
 SYSTEM                          mkdir ${BASEDIR}/OPAM/download-cache/md5-dir/
 SYSTEM                          copy ${OPAMTMP}/v1.0.0.tar.gz -> ${BASEDIR}/OPAM/download-cache/md5-dir/