From bf81a54bc2dee78af20aefdbd6877303a8559e18 Mon Sep 17 00:00:00 2001 From: milner Date: Tue, 12 Jul 2022 08:44:33 +0000 Subject: [PATCH] perf(cache): memoise file digests and invalidate them on write A header included by fourteen translation units was hashed once per dependent, and every built action hashed its inputs twice: once to decide it was stale and again to store the new key. Digests are now memoised for the run, and the scheduler drops the entries for an action's outputs and depfile once it finishes, so a freshly written file is never read from the memo. --- lib/cache.ml | 12 +++++++++++- lib/sched.ml | 3 +++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/lib/cache.ml b/lib/cache.ml index 19607e0..db70a0a 100644 --- a/lib/cache.ml +++ b/lib/cache.ml @@ -24,7 +24,17 @@ let current c out = Hashtbl.find_opt c.keys out let put c out key = Hashtbl.replace c.keys out key let drop c out = Hashtbl.remove c.keys out -let digest_file p = match Digest.file p with d -> Some (Digest.to_hex d) | exception Sys_error _ -> None +let digests : (string, string option) Hashtbl.t = Hashtbl.create 256 + +let digest_file p = + match Hashtbl.find_opt digests p with + | Some d -> d + | None -> + let d = match Digest.file p with d -> Some (Digest.to_hex d) | exception Sys_error _ -> None in + Hashtbl.replace digests p d; + d + +let forget p = Hashtbl.remove digests p let key ~cmd ~inputs = let b = Buffer.create 256 in diff --git a/lib/sched.ml b/lib/sched.ml index e7b184e..16e995b 100644 --- a/lib/sched.ml +++ b/lib/sched.ml @@ -78,11 +78,14 @@ let run g ~selected ~jobs ~cache ~verbose ~keep_going ~on_start ~on_done = if ok then begin incr built; state.(v.id) <- Done; + List.iter Cache.forget v.outs; + (match v.depfile with Some d -> Cache.forget d | None -> ()); Cache.put cache (List.hd v.outs) (key_of v); release v.id end else begin incr failed; + List.iter Cache.forget v.outs; Cache.drop cache (List.hd v.outs); List.iter (fun o -> try Sys.remove o with Sys_error _ -> ()) v.outs; skip v.id