perf(cache): memoise file digests and invalidate them on write

A header included by fourteen translation units was hashed once per dependent,
and every built action hashed its inputs twice: once to decide it was stale and
again to store the new key. Digests are now memoised for the run, and the
scheduler drops the entries for an action's outputs and depfile once it
finishes, so a freshly written file is never read from the memo.

lib/cache.ml

@@ -24,7 +24,17 @@ let current c out = Hashtbl.find_opt c.keys out
 let put c out key = Hashtbl.replace c.keys out key
 let drop c out = Hashtbl.remove c.keys out
 
-let digest_file p = match Digest.file p with d -> Some (Digest.to_hex d) | exception Sys_error _ -> None
+let digests : (string, string option) Hashtbl.t = Hashtbl.create 256
+
+let digest_file p =
+  match Hashtbl.find_opt digests p with
+  | Some d -> d
+  | None ->
+      let d = match Digest.file p with d -> Some (Digest.to_hex d) | exception Sys_error _ -> None in
+      Hashtbl.replace digests p d;
+      d
+
+let forget p = Hashtbl.remove digests p
 
 let key ~cmd ~inputs =
   let b = Buffer.create 256 in

lib/sched.ml

@@ -78,11 +78,14 @@ let run g ~selected ~jobs ~cache ~verbose ~keep_going ~on_start ~on_done =
         if ok then begin
           incr built;
           state.(v.id) <- Done;
+          List.iter Cache.forget v.outs;
+          (match v.depfile with Some d -> Cache.forget d | None -> ());
           Cache.put cache (List.hd v.outs) (key_of v);
           release v.id
         end
         else begin
           incr failed;
+          List.iter Cache.forget v.outs;
           Cache.drop cache (List.hd v.outs);
           List.iter (fun o -> try Sys.remove o with Sys_error _ -> ()) v.outs;
           skip v.id