pkg: add support for opam depexts (#10900)

* pkg: add support for opam depexts

`dune` will just print a warning message whenever a package fails to build.

Signed-off-by: Alpha DIALLO <moyodiallo@gmail.com>

src/dune_pkg/lock_dir.ml

@@ -105,37 +105,42 @@ module Pkg = struct
     { build_command : Build_command.t option
     ; install_command : Action.t option
     ; depends : (Loc.t * Package_name.t) list
+    ; depexts : string list
     ; info : Pkg_info.t
     ; exported_env : String_with_vars.t Action.Env_update.t list
     }
 
-  let equal { build_command; install_command; depends; info; exported_env } t =
+  let equal { build_command; install_command; depends; depexts; info; exported_env } t =
     Option.equal Build_command.equal build_command t.build_command
     (* CR-rgrinberg: why do we ignore locations? *)
     && Option.equal Action.equal_no_locs install_command t.install_command
     && List.equal (Tuple.T2.equal Loc.equal Package_name.equal) depends t.depends
+    && List.equal String.equal depexts t.depexts
     && Pkg_info.equal info t.info
     && List.equal
          (Action.Env_update.equal String_with_vars.equal)
          exported_env
          t.exported_env
   ;;
 
-  let remove_locs { build_command; install_command; depends; info; exported_env } =
+  let remove_locs { build_command; install_command; depends; depexts; info; exported_env }
+    =
     { info = Pkg_info.remove_locs info
     ; exported_env =
         List.map exported_env ~f:(Action.Env_update.map ~f:String_with_vars.remove_locs)
     ; depends = List.map depends ~f:(fun (_, pkg) -> Loc.none, pkg)
+    ; depexts
     ; build_command = Option.map build_command ~f:Build_command.remove_locs
     ; install_command = Option.map install_command ~f:Action.remove_locs
     }
   ;;
 
-  let to_dyn { build_command; install_command; depends; info; exported_env } =
+  let to_dyn { build_command; install_command; depends; depexts; info; exported_env } =
     Dyn.record
       [ "build_command", Dyn.option Build_command.to_dyn build_command
       ; "install_command", Dyn.option Action.to_dyn install_command
       ; "depends", Dyn.list (Dyn.pair Loc.to_dyn_hum Package_name.to_dyn) depends
+      ; "depexts", Dyn.list String.to_dyn depexts
       ; "info", Pkg_info.to_dyn info
       ; ( "exported_env"
         , Dyn.list (Action.Env_update.to_dyn String_with_vars.to_dyn) exported_env )
@@ -157,6 +162,7 @@ module Pkg = struct
     let version = "version"
     let install = "install"
     let depends = "depends"
+    let depexts = "depexts"
     let source = "source"
     let dev = "dev"
     let exported_env = "exported_env"
@@ -172,6 +178,7 @@ module Pkg = struct
        and+ build_command = Build_command.decode
        and+ depends =
          field ~default:[] Fields.depends (repeat (located Package_name.decode))
+       and+ depexts = field ~default:[] Fields.depexts (repeat string)
        and+ source = field_o Fields.source Source.decode
        and+ dev = field_b Fields.dev
        and+ exported_env =
@@ -196,7 +203,7 @@ module Pkg = struct
            in
            { Pkg_info.name; version; dev; source; extra_sources }
          in
-         { build_command; depends; install_command; info; exported_env }
+         { build_command; depends; depexts; install_command; info; exported_env }
   ;;
 
   let encode_extra_source (local, source) : Dune_sexp.t =
@@ -210,6 +217,7 @@ module Pkg = struct
     { build_command
     ; install_command
     ; depends
+    ; depexts
     ; info = { Pkg_info.name = _; extra_sources; version; dev; source }
     ; exported_env
     }
@@ -220,6 +228,7 @@ module Pkg = struct
       ; field_o Fields.install Action.encode install_command
       ; Build_command.encode build_command
       ; field_l Fields.depends Package_name.encode (List.map depends ~f:snd)
+      ; field_l Fields.depexts string depexts
       ; field_o Fields.source Source.encode source
       ; field_b Fields.dev dev
       ; field_l Fields.exported_env Action.Env_update.encode exported_env

src/dune_pkg/lock_dir.mli

@@ -26,6 +26,7 @@ module Pkg : sig
     { build_command : Build_command.t option
     ; install_command : Action.t option
     ; depends : (Loc.t * Package_name.t) list
+    ; depexts : string list
     ; info : Pkg_info.t
     ; exported_env : String_with_vars.t Action.Env_update.t list
     }

src/dune_pkg/opam_solver.ml

@@ -704,6 +704,13 @@ let opam_package_to_lock_file_pkg
       |> Option.map ~f:build_env
       |> Option.map ~f:(fun action -> Lock_dir.Build_command.Action action))
   in
+  let depexts =
+    OpamFile.OPAM.depexts opam_file
+    |> List.concat_map ~f:(fun (sys_pkgs, filter) ->
+      if OpamFilter.eval_to_bool (Solver_env.to_env solver_env) filter
+      then OpamSysPkg.Set.to_list_map OpamSysPkg.to_string sys_pkgs
+      else [])
+  in
   let install_command =
     OpamFile.OPAM.install opam_file
     |> opam_commands_to_actions get_solver_var loc opam_package
@@ -714,7 +721,9 @@ let opam_package_to_lock_file_pkg
     OpamFile.OPAM.env opam_file |> List.map ~f:opam_env_update_to_env_update
   in
   let kind = if opam_file_is_compiler opam_file then `Compiler else `Non_compiler in
-  kind, { Lock_dir.Pkg.build_command; install_command; depends; info; exported_env }
+  ( kind
+  , { Lock_dir.Pkg.build_command; install_command; depends; depexts; info; exported_env }
+  )
 ;;
 
 let solve_package_list packages ~context =

src/dune_rules/pkg_rules.ml

@@ -327,6 +327,7 @@ module Pkg = struct
     ; build_command : Build_command.t option
     ; install_command : Dune_lang.Action.t option
     ; depends : t list
+    ; depexts : string list
     ; info : Pkg_info.t
     ; paths : Path.t Paths.t
     ; write_paths : Path.Build.t Paths.t
@@ -482,6 +483,7 @@ module Expander0 = struct
     ; artifacts : Path.t Filename.Map.t
     ; depends :
         (Variable.value Package_variable_name.Map.t * Path.t Paths.t) Package.Name.Map.t
+    ; depexts : string list
     ; context : Context_name.t
     ; version : Package_version.t
     ; env : Value.t list Env.Map.t
@@ -586,6 +588,7 @@ module Run_with_path = struct
     val with_error
       :  accepted_exit_codes:int Predicate.t
       -> pkg:Dune_pkg.Package_name.t * Loc.t
+      -> depexts:string list
       -> display:Display.t
       -> (error -> 'a)
       -> 'a
@@ -594,6 +597,7 @@ module Run_with_path = struct
   end = struct
     type error =
       { pkg : Dune_pkg.Package_name.t * Loc.t
+      ; depexts : string list
       ; filename : Dpath.t
       ; io : Process.Io.output Process.Io.t
       ; accepted_exit_codes : int Predicate.t
@@ -602,22 +606,27 @@ module Run_with_path = struct
 
     let io t = t.io
 
-    let with_error ~accepted_exit_codes ~pkg ~display f =
+    let with_error ~accepted_exit_codes ~pkg ~depexts ~display f =
       let filename = Temp.create File ~prefix:"dune-pkg" ~suffix:"stderr" in
       let io = Process.Io.(file filename Out) in
-      let t = { filename; io; accepted_exit_codes; display; pkg } in
+      let t = { filename; io; accepted_exit_codes; display; pkg; depexts } in
       let result = f t in
       Temp.destroy File filename;
       result
     ;;
 
     let to_paragraphs t error =
-      let pp_pkg =
-        let pkg_name = Dune_pkg.Package_name.to_string (fst t.pkg) in
-        Pp.textf "Logs for package %s" pkg_name
+      let pkg_name, loc = t.pkg in
+      let depexts_warning =
+        match t.depexts with
+        | [] -> []
+        | _ :: _ ->
+          [ Pp.textf "You may want to verify the following depexts are installed:"
+          ; Pp.enumerate ~f:Pp.verbatim t.depexts
+          ]
       in
-      let loc = snd t.pkg in
-      [ pp_pkg; Pp.verbatim error ], loc
+      let pp_pkg = Pp.textf "Logs for package %s" (Package.Name.to_string pkg_name) in
+      [ pp_pkg; Pp.verbatim error ] @ depexts_warning, loc
     ;;
 
     let prerr ~rc error =
@@ -647,6 +656,7 @@ module Run_with_path = struct
       ; args : 'path arg Array.Immutable.t
       ; ocamlfind_destdir : 'path
       ; pkg : Dune_pkg.Package_name.t * Loc.t
+      ; depexts : string list
       }
 
     let name = "run-with-path"
@@ -667,7 +677,7 @@ module Run_with_path = struct
 
     let is_useful_to ~memoize:_ = true
 
-    let encode { prog; args; ocamlfind_destdir; pkg = _ } path _ : Sexp.t =
+    let encode { prog; args; ocamlfind_destdir; pkg = _; depexts = _ } path _ : Sexp.t =
       let prog : Sexp.t =
         Atom
           (match prog with
@@ -685,7 +695,7 @@ module Run_with_path = struct
     ;;
 
     let action
-      { prog; args; ocamlfind_destdir; pkg }
+      { prog; args; ocamlfind_destdir; pkg; depexts }
       ~(ectx : Action.context)
       ~(eenv : Action.env)
       =
@@ -708,33 +718,38 @@ module Run_with_path = struct
             ~var:"OCAMLFIND_DESTDIR"
             ~value:(Path.to_absolute_filename ocamlfind_destdir)
         in
-        Output.with_error ~accepted_exit_codes:eenv.exit_codes ~pkg ~display (fun error ->
-          let stdout_to =
-            match !Clflags.debug_package_logs, display with
-            | true, _ | false, Display.Verbose -> eenv.stdout_to
-            | _ -> Process.Io.(null Out)
-          in
-          Process.run
-            Return
-            prog
-            args
-            ~display
-            ~metadata
-            ~stdout_to
-            ~stderr_to:(Output.io error)
-            ~stdin_from:eenv.stdin_from
-            ~dir:eenv.working_dir
-            ~env
-          >>= fun (_, rc) ->
-          Output.prerr ~rc error;
-          Fiber.return ())
+        Output.with_error
+          ~accepted_exit_codes:eenv.exit_codes
+          ~pkg
+          ~depexts
+          ~display
+          (fun error ->
+             let stdout_to =
+               match !Clflags.debug_package_logs, display with
+               | true, _ | false, Display.Verbose -> eenv.stdout_to
+               | _ -> Process.Io.(null Out)
+             in
+             Process.run
+               Return
+               prog
+               args
+               ~display
+               ~metadata
+               ~stdout_to
+               ~stderr_to:(Output.io error)
+               ~stdin_from:eenv.stdin_from
+               ~dir:eenv.working_dir
+               ~env
+             >>= fun (_, rc) ->
+             Output.prerr ~rc error;
+             Fiber.return ())
     ;;
   end
 
   module A = Action_ext.Make (Spec)
 
-  let action ~pkg prog args ~ocamlfind_destdir =
-    A.action { Spec.prog; args; ocamlfind_destdir; pkg }
+  let action ~pkg ~depexts prog args ~ocamlfind_destdir =
+    A.action { Spec.prog; args; ocamlfind_destdir; pkg; depexts }
   ;;
 end
 
@@ -858,7 +873,15 @@ module Action_expander = struct
     ;;
 
     let expand_pform
-      { name = _; env = _; paths; artifacts = _; context; depends; version = _ }
+      { name = _
+      ; env = _
+      ; paths
+      ; artifacts = _
+      ; context
+      ; depends
+      ; version = _
+      ; depexts = _
+      }
       ~source
       (pform : Pform.t)
       : (Value.t list, [ `Undefined_pkg_var of Package_variable_name.t ]) result Memo.t
@@ -983,7 +1006,12 @@ module Action_expander = struct
                | Path p | Dir p -> Path p))
          in
          let ocamlfind_destdir = (Lazy.force expander.paths.install_roots).lib_root in
-         Run_with_path.action ~pkg:(expander.name, prog_loc) exe args ~ocamlfind_destdir)
+         Run_with_path.action
+           ~depexts:expander.depexts
+           ~pkg:(expander.name, prog_loc)
+           exe
+           args
+           ~ocamlfind_destdir)
     | Progn t ->
       let+ args = Memo.parallel_map t ~f:(expand ~expander) in
       Action.Progn args
@@ -1116,6 +1144,7 @@ module Action_expander = struct
     ; artifacts = binaries
     ; context
     ; depends
+    ; depexts = pkg.depexts
     ; version = pkg.info.version
     ; env
     }
@@ -1218,8 +1247,13 @@ end = struct
     match Package.Name.Map.find db.all name with
     | None -> Memo.return None
     | Some
-        ({ Lock_dir.Pkg.build_command; install_command; depends; info; exported_env } as
-         pkg) ->
+        ({ Lock_dir.Pkg.build_command
+         ; install_command
+         ; depends
+         ; info
+         ; exported_env
+         ; depexts
+         } as pkg) ->
       assert (Package.Name.equal name info.name);
       let* depends =
         Memo.parallel_map depends ~f:(fun name ->
@@ -1276,6 +1310,7 @@ end = struct
         ; build_command
         ; install_command
         ; depends
+        ; depexts
         ; paths
         ; write_paths
         ; info

test/blackbox-tests/test-cases/pkg/depexts/error-message.t

@@ -0,0 +1,53 @@
+ When a package fails to build, dune will print opam depexts warning.
+
+  $ . ../helpers.sh
+  $ mkrepo
+  $ add_mock_repo_if_needed
+
+Make a library that would fail when building it:
+  $ mkdir foo
+  $ cat > foo/dune-project <<EOF
+  > EOF
+  $ tar -czf foo.tar.gz foo
+  $ rm -rf foo
+
+Make a project that uses the foo library:
+  $ cat > dune-project <<EOF
+  > (lang dune 3.13)
+  > (package
+  >  (name bar)
+  >  (depends foo))
+  > EOF
+  $ cat > dune <<EOF
+  > (executable
+  >  (public_name bar)
+  >  (libraries foo))
+  > EOF
+
+Make dune.lock files
+  $ make_lockdir
+  $ cat > dune.lock/foo.pkg <<EOF
+  > (version 0.0.1)
+  > (build
+  >  (run dune build))
+  > (depexts unzip gnupg)
+  > (source
+  >  (fetch
+  >   (url file://$PWD/foo.tar.gz)
+  >   (checksum md5=$(md5sum foo.tar.gz | cut -f1 -d' '))))
+  > EOF
+
+Build the project, when it fails building 'foo' package, it shows
+the depexts error message.
+  $ dune build
+  File "dune.lock/foo.pkg", line 3, characters 6-10:
+  3 |  (run dune build))
+            ^^^^
+  Error: Logs for package foo
+  File "dune-project", line 1, characters 0-0:
+  Error: Invalid first line, expected: (lang <lang> <version>)
+
+  You may want to verify the following depexts are installed:
+  - unzip
+  - gnupg
+  [1]