From 1b2deaf57aeec36ede38a81184fbafebb64fd4ac Mon Sep 17 00:00:00 2001 From: Camille GILLOT Date: Fri, 6 Mar 2020 22:15:46 +0100 Subject: [PATCH 1/9] Monomorphise force_query_with_job. --- src/librustc_query_system/query/config.rs | 25 ++++++++++++++++++ src/librustc_query_system/query/plumbing.rs | 29 ++++++++++++--------- 2 files changed, 41 insertions(+), 13 deletions(-) diff --git a/src/librustc_query_system/query/config.rs b/src/librustc_query_system/query/config.rs index 710ec3bfb0b3c..37a304cb5071f 100644 --- a/src/librustc_query_system/query/config.rs +++ b/src/librustc_query_system/query/config.rs @@ -24,6 +24,15 @@ pub trait QueryConfig { type Stored: Clone; } +pub(crate) struct QueryVtable { + pub eval_always: bool, + + // Don't use this method to compute query results, instead use the methods on TyCtxt + pub compute: fn(CTX, K) -> V, + + pub hash_result: fn(&mut CTX::StableHashingContext, &V) -> Option, +} + pub trait QueryAccessors: QueryConfig { const ANON: bool; const EVAL_ALWAYS: bool; @@ -60,6 +69,22 @@ pub trait QueryDescription: QueryAccessors { } } +pub(crate) trait QueryVtableExt { + const VTABLE: QueryVtable; +} + +impl QueryVtableExt for Q +where + CTX: QueryContext, + Q: QueryDescription, +{ + const VTABLE: QueryVtable = QueryVtable { + eval_always: Q::EVAL_ALWAYS, + compute: Q::compute, + hash_result: Q::hash_result, + }; +} + impl QueryDescription for M where M: QueryAccessors, diff --git a/src/librustc_query_system/query/plumbing.rs b/src/librustc_query_system/query/plumbing.rs index 0aeec269e617a..21cd7ce567a2b 100644 --- a/src/librustc_query_system/query/plumbing.rs +++ b/src/librustc_query_system/query/plumbing.rs @@ -5,7 +5,7 @@ use crate::dep_graph::{DepKind, DepNode}; use crate::dep_graph::{DepNodeIndex, SerializedDepNodeIndex}; use crate::query::caches::QueryCache; -use crate::query::config::QueryDescription; +use crate::query::config::{QueryDescription, QueryVtable, QueryVtableExt}; use crate::query::job::{QueryInfo, QueryJob, QueryJobId, QueryJobInfo, QueryShardJobId}; use crate::query::QueryContext; @@ -406,7 +406,7 @@ where // expensive for some `DepKind`s. if !tcx.dep_graph().is_fully_enabled() { let null_dep_node = DepNode::new_no_params(DepKind::NULL); - return force_query_with_job::(tcx, key, job, null_dep_node).0; + return force_query_with_job(tcx, key, job, null_dep_node, &Q::VTABLE).0; } if Q::ANON { @@ -455,7 +455,7 @@ where } } - let (result, dep_node_index) = force_query_with_job::(tcx, key, job, dep_node); + let (result, dep_node_index) = force_query_with_job(tcx, key, job, dep_node, &Q::VTABLE); tcx.dep_graph().read_index(dep_node_index); result } @@ -549,14 +549,17 @@ fn incremental_verify_ich( } #[inline(always)] -fn force_query_with_job( +fn force_query_with_job( tcx: CTX, - key: Q::Key, - job: JobOwner<'_, CTX, Q::Cache>, + key: C::Key, + job: JobOwner<'_, CTX, C>, dep_node: DepNode, -) -> (Q::Stored, DepNodeIndex) + query: &QueryVtable, +) -> (C::Stored, DepNodeIndex) where - Q: QueryDescription, + C: QueryCache, + C::Key: Eq + Clone + Debug, + C::Stored: Clone, CTX: QueryContext, { // If the following assertion triggers, it can have two reasons: @@ -577,16 +580,16 @@ where let ((result, dep_node_index), diagnostics) = with_diagnostics(|diagnostics| { tcx.start_query(job.id, diagnostics, |tcx| { - if Q::EVAL_ALWAYS { + if query.eval_always { tcx.dep_graph().with_eval_always_task( dep_node, tcx, key, - Q::compute, - Q::hash_result, + query.compute, + query.hash_result, ) } else { - tcx.dep_graph().with_task(dep_node, tcx, key, Q::compute, Q::hash_result) + tcx.dep_graph().with_task(dep_node, tcx, key, query.compute, query.hash_result) } }) }); @@ -684,7 +687,7 @@ where #[cfg(parallel_compiler)] TryGetJob::JobCompleted(_) => return, }; - force_query_with_job::(tcx, key, job, dep_node); + force_query_with_job(tcx, key, job, dep_node, &Q::VTABLE); }, ); } From 85704a41dbba580a11c2ed59e8e306ec5a82c8f0 Mon Sep 17 00:00:00 2001 From: Camille GILLOT Date: Fri, 6 Mar 2020 22:43:08 +0100 Subject: [PATCH 2/9] Monomorphise load_from_disk_and_cache_in_memory. --- src/librustc_macros/src/query.rs | 4 +-- src/librustc_middle/ty/query/on_disk_cache.rs | 2 +- src/librustc_query_system/query/config.rs | 30 +++++++++++++++++-- src/librustc_query_system/query/plumbing.rs | 27 +++++++++-------- 4 files changed, 45 insertions(+), 18 deletions(-) diff --git a/src/librustc_macros/src/query.rs b/src/librustc_macros/src/query.rs index 229ccfe08f7af..5f5bae66cfc65 100644 --- a/src/librustc_macros/src/query.rs +++ b/src/librustc_macros/src/query.rs @@ -365,7 +365,7 @@ fn add_query_description_impl( #[allow(unused_variables, unused_braces)] fn cache_on_disk( #tcx: TyCtxt<'tcx>, - #key: Self::Key, + #key: &Self::Key, #value: Option<&Self::Value> ) -> bool { #expr @@ -441,7 +441,7 @@ pub fn rustc_queries(input: TokenStream) -> TokenStream { .unwrap_or(false)); let key = <#arg as DepNodeParams>>::recover($tcx, $dep_node).unwrap(); - if queries::#name::cache_on_disk($tcx, key, None) { + if queries::#name::cache_on_disk($tcx, &key, None) { let _ = $tcx.#name(key); } } diff --git a/src/librustc_middle/ty/query/on_disk_cache.rs b/src/librustc_middle/ty/query/on_disk_cache.rs index e9bb659d5c839..760fdbe8522f4 100644 --- a/src/librustc_middle/ty/query/on_disk_cache.rs +++ b/src/librustc_middle/ty/query/on_disk_cache.rs @@ -1009,7 +1009,7 @@ where state.iter_results(|results| { for (key, value, dep_node) in results { - if Q::cache_on_disk(tcx, key.clone(), Some(&value)) { + if Q::cache_on_disk(tcx, &key, Some(&value)) { let dep_node = SerializedDepNodeIndex::new(dep_node.index()); // Record position of the cache entry. diff --git a/src/librustc_query_system/query/config.rs b/src/librustc_query_system/query/config.rs index 37a304cb5071f..7a13bbf22999d 100644 --- a/src/librustc_query_system/query/config.rs +++ b/src/librustc_query_system/query/config.rs @@ -31,6 +31,30 @@ pub(crate) struct QueryVtable { pub compute: fn(CTX, K) -> V, pub hash_result: fn(&mut CTX::StableHashingContext, &V) -> Option, + pub cache_on_disk: fn(CTX, &K, Option<&V>) -> bool, + pub try_load_from_disk: fn(CTX, SerializedDepNodeIndex) -> Option, +} + +impl QueryVtable { + pub(crate) fn compute(&self, tcx: CTX, key: K) -> V { + (self.compute)(tcx, key) + } + + pub(crate) fn hash_result( + &self, + hcx: &mut CTX::StableHashingContext, + value: &V, + ) -> Option { + (self.hash_result)(hcx, value) + } + + pub(crate) fn cache_on_disk(&self, tcx: CTX, key: &K, value: Option<&V>) -> bool { + (self.cache_on_disk)(tcx, key, value) + } + + pub(crate) fn try_load_from_disk(&self, tcx: CTX, index: SerializedDepNodeIndex) -> Option { + (self.try_load_from_disk)(tcx, index) + } } pub trait QueryAccessors: QueryConfig { @@ -60,7 +84,7 @@ pub trait QueryDescription: QueryAccessors { fn describe(tcx: CTX, key: Self::Key) -> Cow<'static, str>; #[inline] - fn cache_on_disk(_: CTX, _: Self::Key, _: Option<&Self::Value>) -> bool { + fn cache_on_disk(_: CTX, _: &Self::Key, _: Option<&Self::Value>) -> bool { false } @@ -82,6 +106,8 @@ where eval_always: Q::EVAL_ALWAYS, compute: Q::compute, hash_result: Q::hash_result, + cache_on_disk: Q::cache_on_disk, + try_load_from_disk: Q::try_load_from_disk, }; } @@ -98,7 +124,7 @@ where } } - default fn cache_on_disk(_: CTX, _: Self::Key, _: Option<&Self::Value>) -> bool { + default fn cache_on_disk(_: CTX, _: &Self::Key, _: Option<&Self::Value>) -> bool { false } diff --git a/src/librustc_query_system/query/plumbing.rs b/src/librustc_query_system/query/plumbing.rs index 21cd7ce567a2b..4a03c172d3cda 100644 --- a/src/librustc_query_system/query/plumbing.rs +++ b/src/librustc_query_system/query/plumbing.rs @@ -439,12 +439,13 @@ where let marked = tcx.dep_graph().try_mark_green_and_read(tcx, &dep_node); marked.map(|(prev_dep_node_index, dep_node_index)| { ( - load_from_disk_and_cache_in_memory::( + load_from_disk_and_cache_in_memory( tcx, key.clone(), prev_dep_node_index, dep_node_index, &dep_node, + &Q::VTABLE, ), dep_node_index, ) @@ -460,16 +461,16 @@ where result } -fn load_from_disk_and_cache_in_memory( +fn load_from_disk_and_cache_in_memory( tcx: CTX, - key: Q::Key, + key: K, prev_dep_node_index: SerializedDepNodeIndex, dep_node_index: DepNodeIndex, dep_node: &DepNode, -) -> Q::Value + query: &QueryVtable, +) -> V where CTX: QueryContext, - Q: QueryDescription, { // Note this function can be called concurrently from the same query // We must ensure that this is handled correctly. @@ -477,9 +478,9 @@ where debug_assert!(tcx.dep_graph().is_green(dep_node)); // First we try to load the result from the on-disk cache. - let result = if Q::cache_on_disk(tcx, key.clone(), None) { + let result = if query.cache_on_disk(tcx, &key, None) { let prof_timer = tcx.profiler().incr_cache_loading(); - let result = Q::try_load_from_disk(tcx, prev_dep_node_index); + let result = query.try_load_from_disk(tcx, prev_dep_node_index); prof_timer.finish_with_query_invocation_id(dep_node_index.into()); // We always expect to find a cached result for things that @@ -503,7 +504,7 @@ where let prof_timer = tcx.profiler().query_provider(); // The dep-graph for this computation is already in-place. - let result = tcx.dep_graph().with_ignore(|| Q::compute(tcx, key)); + let result = tcx.dep_graph().with_ignore(|| query.compute(tcx, key)); prof_timer.finish_with_query_invocation_id(dep_node_index.into()); @@ -513,7 +514,7 @@ where // If `-Zincremental-verify-ich` is specified, re-hash results from // the cache and make sure that they have the expected fingerprint. if unlikely!(tcx.incremental_verify_ich()) { - incremental_verify_ich::(tcx, &result, dep_node, dep_node_index); + incremental_verify_ich(tcx, &result, dep_node, dep_node_index, query); } result @@ -521,14 +522,14 @@ where #[inline(never)] #[cold] -fn incremental_verify_ich( +fn incremental_verify_ich( tcx: CTX, - result: &Q::Value, + result: &V, dep_node: &DepNode, dep_node_index: DepNodeIndex, + query: &QueryVtable, ) where CTX: QueryContext, - Q: QueryDescription, { assert!( Some(tcx.dep_graph().fingerprint_of(dep_node_index)) @@ -540,7 +541,7 @@ fn incremental_verify_ich( debug!("BEGIN verify_ich({:?})", dep_node); let mut hcx = tcx.create_stable_hashing_context(); - let new_hash = Q::hash_result(&mut hcx, result).unwrap_or(Fingerprint::ZERO); + let new_hash = query.hash_result(&mut hcx, result).unwrap_or(Fingerprint::ZERO); debug!("END verify_ich({:?})", dep_node); let old_hash = tcx.dep_graph().fingerprint_of(dep_node_index); From d56085cbc96aafe9ac2403201edc0a9728a4fea8 Mon Sep 17 00:00:00 2001 From: Camille GILLOT Date: Fri, 6 Mar 2020 22:56:05 +0100 Subject: [PATCH 3/9] Monomorphise try_execute_anon_query. --- src/librustc_query_system/query/config.rs | 4 ++ src/librustc_query_system/query/plumbing.rs | 45 ++++++++++++++------- 2 files changed, 34 insertions(+), 15 deletions(-) diff --git a/src/librustc_query_system/query/config.rs b/src/librustc_query_system/query/config.rs index 7a13bbf22999d..baf9ca9df7fb8 100644 --- a/src/librustc_query_system/query/config.rs +++ b/src/librustc_query_system/query/config.rs @@ -25,6 +25,8 @@ pub trait QueryConfig { } pub(crate) struct QueryVtable { + pub anon: bool, + pub dep_kind: CTX::DepKind, pub eval_always: bool, // Don't use this method to compute query results, instead use the methods on TyCtxt @@ -103,6 +105,8 @@ where Q: QueryDescription, { const VTABLE: QueryVtable = QueryVtable { + anon: Q::ANON, + dep_kind: Q::DEP_KIND, eval_always: Q::EVAL_ALWAYS, compute: Q::compute, hash_result: Q::hash_result, diff --git a/src/librustc_query_system/query/plumbing.rs b/src/librustc_query_system/query/plumbing.rs index 4a03c172d3cda..f27c508fccfee 100644 --- a/src/librustc_query_system/query/plumbing.rs +++ b/src/librustc_query_system/query/plumbing.rs @@ -410,21 +410,7 @@ where } if Q::ANON { - let prof_timer = tcx.profiler().query_provider(); - - let ((result, dep_node_index), diagnostics) = with_diagnostics(|diagnostics| { - tcx.start_query(job.id, diagnostics, |tcx| { - tcx.dep_graph().with_anon_task(Q::DEP_KIND, || Q::compute(tcx, key)) - }) - }); - - prof_timer.finish_with_query_invocation_id(dep_node_index.into()); - - tcx.dep_graph().read_index(dep_node_index); - - if unlikely!(!diagnostics.is_empty()) { - tcx.store_diagnostics_for_anon_node(dep_node_index, diagnostics); - } + let (result, dep_node_index) = try_execute_anon_query(tcx, key, job.id, &Q::VTABLE); return job.complete(tcx, result, dep_node_index); } @@ -461,6 +447,35 @@ where result } +fn try_execute_anon_query( + tcx: CTX, + key: K, + job_id: QueryJobId, + query: &QueryVtable, +) -> (V, DepNodeIndex) +where + CTX: QueryContext, +{ + debug_assert!(query.anon); + let prof_timer = tcx.profiler().query_provider(); + + let ((result, dep_node_index), diagnostics) = with_diagnostics(|diagnostics| { + tcx.start_query(job_id, diagnostics, |tcx| { + tcx.dep_graph().with_anon_task(query.dep_kind, || query.compute(tcx, key)) + }) + }); + + prof_timer.finish_with_query_invocation_id(dep_node_index.into()); + + tcx.dep_graph().read_index(dep_node_index); + + if unlikely!(!diagnostics.is_empty()) { + tcx.store_diagnostics_for_anon_node(dep_node_index, diagnostics); + } + + (result, dep_node_index) +} + fn load_from_disk_and_cache_in_memory( tcx: CTX, key: K, From 1c7376e7979ee8179c030d101af2d7919369caa7 Mon Sep 17 00:00:00 2001 From: Camille GILLOT Date: Sat, 7 Mar 2020 17:32:07 +0100 Subject: [PATCH 4/9] Monomorphise try_start. --- src/librustc_query_system/query/config.rs | 6 ++++ src/librustc_query_system/query/plumbing.rs | 38 +++++++++++---------- 2 files changed, 26 insertions(+), 18 deletions(-) diff --git a/src/librustc_query_system/query/config.rs b/src/librustc_query_system/query/config.rs index baf9ca9df7fb8..48fbdfb153eee 100644 --- a/src/librustc_query_system/query/config.rs +++ b/src/librustc_query_system/query/config.rs @@ -33,6 +33,7 @@ pub(crate) struct QueryVtable { pub compute: fn(CTX, K) -> V, pub hash_result: fn(&mut CTX::StableHashingContext, &V) -> Option, + pub handle_cycle_error: fn(CTX, CycleError) -> V, pub cache_on_disk: fn(CTX, &K, Option<&V>) -> bool, pub try_load_from_disk: fn(CTX, SerializedDepNodeIndex) -> Option, } @@ -50,6 +51,10 @@ impl QueryVtable { (self.hash_result)(hcx, value) } + pub(crate) fn handle_cycle_error(&self, tcx: CTX, error: CycleError) -> V { + (self.handle_cycle_error)(tcx, error) + } + pub(crate) fn cache_on_disk(&self, tcx: CTX, key: &K, value: Option<&V>) -> bool { (self.cache_on_disk)(tcx, key, value) } @@ -110,6 +115,7 @@ where eval_always: Q::EVAL_ALWAYS, compute: Q::compute, hash_result: Q::hash_result, + handle_cycle_error: Q::handle_cycle_error, cache_on_disk: Q::cache_on_disk, try_load_from_disk: Q::try_load_from_disk, }; diff --git a/src/librustc_query_system/query/plumbing.rs b/src/librustc_query_system/query/plumbing.rs index f27c508fccfee..ff01538d95e4c 100644 --- a/src/librustc_query_system/query/plumbing.rs +++ b/src/librustc_query_system/query/plumbing.rs @@ -168,14 +168,15 @@ where /// This function is inlined because that results in a noticeable speed-up /// for some compile-time benchmarks. #[inline(always)] - fn try_start<'a, 'b, Q>( + fn try_start<'a, 'b>( tcx: CTX, + state: &'b QueryState, span: Span, key: &C::Key, mut lookup: QueryLookup<'a, CTX, C::Key, C::Sharded>, + query: &QueryVtable, ) -> TryGetJob<'b, CTX, C> where - Q: QueryDescription, CTX: QueryContext, { let lock = &mut *lookup.lock; @@ -194,7 +195,7 @@ where }; // Create the id of the job we're waiting for - let id = QueryJobId::new(job.id, lookup.shard, Q::DEP_KIND); + let id = QueryJobId::new(job.id, lookup.shard, query.dep_kind); (job.latch(id), _query_blocked_prof_timer) } @@ -209,15 +210,14 @@ where lock.jobs = id; let id = QueryShardJobId(NonZeroU32::new(id).unwrap()); - let global_id = QueryJobId::new(id, lookup.shard, Q::DEP_KIND); + let global_id = QueryJobId::new(id, lookup.shard, query.dep_kind); let job = tcx.current_query_job(); let job = QueryJob::new(id, span, job); entry.insert(QueryResult::Started(job)); - let owner = - JobOwner { state: Q::query_state(tcx), id: global_id, key: (*key).clone() }; + let owner = JobOwner { state, id: global_id, key: (*key).clone() }; return TryGetJob::NotYetStarted(owner); } }; @@ -227,8 +227,8 @@ where // so we just return the error. #[cfg(not(parallel_compiler))] return TryGetJob::Cycle(cold_path(|| { - let value = Q::handle_cycle_error(tcx, latch.find_cycle_in_stack(tcx, span)); - Q::query_state(tcx).cache.store_nocache(value) + let value = query.handle_cycle_error(tcx, latch.find_cycle_in_stack(tcx, span)); + state.cache.store_nocache(value) })); // With parallel queries we might just have to wait on some other @@ -238,14 +238,14 @@ where let result = latch.wait_on(tcx, span); if let Err(cycle) = result { - let value = Q::handle_cycle_error(tcx, cycle); - let value = Q::query_state(tcx).cache.store_nocache(value); + let value = query.handle_cycle_error(tcx, cycle); + let value = state.cache.store_nocache(value); return TryGetJob::Cycle(value); } let cached = try_get_cached( tcx, - Q::query_state(tcx), + state, (*key).clone(), |value, index| (value.clone(), index), |_, _| panic!("value must be in cache after waiting"), @@ -392,7 +392,7 @@ where Q: QueryDescription, CTX: QueryContext, { - let job = match JobOwner::try_start::(tcx, span, &key, lookup) { + let job = match JobOwner::try_start(tcx, Q::query_state(tcx), span, &key, lookup, &Q::VTABLE) { TryGetJob::NotYetStarted(job) => job, TryGetJob::Cycle(result) => return result, #[cfg(parallel_compiler)] @@ -697,12 +697,14 @@ where // Cache hit, do nothing }, |key, lookup| { - let job = match JobOwner::try_start::(tcx, span, &key, lookup) { - TryGetJob::NotYetStarted(job) => job, - TryGetJob::Cycle(_) => return, - #[cfg(parallel_compiler)] - TryGetJob::JobCompleted(_) => return, - }; + let job = + match JobOwner::try_start(tcx, Q::query_state(tcx), span, &key, lookup, &Q::VTABLE) + { + TryGetJob::NotYetStarted(job) => job, + TryGetJob::Cycle(_) => return, + #[cfg(parallel_compiler)] + TryGetJob::JobCompleted(_) => return, + }; force_query_with_job(tcx, key, job, dep_node, &Q::VTABLE); }, ); From 8f3e96d6589fad610107fa05ba80fc708965561b Mon Sep 17 00:00:00 2001 From: Camille GILLOT Date: Sat, 28 Mar 2020 13:12:20 +0100 Subject: [PATCH 5/9] Monomorphise try_execute_query. --- src/librustc_query_system/query/config.rs | 6 ++ src/librustc_query_system/query/plumbing.rs | 75 +++++++++------------ 2 files changed, 38 insertions(+), 43 deletions(-) diff --git a/src/librustc_query_system/query/config.rs b/src/librustc_query_system/query/config.rs index 48fbdfb153eee..a334e897e400e 100644 --- a/src/librustc_query_system/query/config.rs +++ b/src/librustc_query_system/query/config.rs @@ -28,6 +28,7 @@ pub(crate) struct QueryVtable { pub anon: bool, pub dep_kind: CTX::DepKind, pub eval_always: bool, + pub to_dep_node: fn(CTX, &K) -> DepNode, // Don't use this method to compute query results, instead use the methods on TyCtxt pub compute: fn(CTX, K) -> V, @@ -39,6 +40,10 @@ pub(crate) struct QueryVtable { } impl QueryVtable { + pub(crate) fn to_dep_node(&self, tcx: CTX, key: &K) -> DepNode { + (self.to_dep_node)(tcx, key) + } + pub(crate) fn compute(&self, tcx: CTX, key: K) -> V { (self.compute)(tcx, key) } @@ -112,6 +117,7 @@ where const VTABLE: QueryVtable = QueryVtable { anon: Q::ANON, dep_kind: Q::DEP_KIND, + to_dep_node: Q::to_dep_node, eval_always: Q::EVAL_ALWAYS, compute: Q::compute, hash_result: Q::hash_result, diff --git a/src/librustc_query_system/query/plumbing.rs b/src/librustc_query_system/query/plumbing.rs index ff01538d95e4c..74a2288cfa790 100644 --- a/src/librustc_query_system/query/plumbing.rs +++ b/src/librustc_query_system/query/plumbing.rs @@ -382,17 +382,21 @@ where } #[inline(always)] -fn try_execute_query( +fn try_execute_query( tcx: CTX, + state: &QueryState, span: Span, - key: Q::Key, - lookup: QueryLookup<'_, CTX, Q::Key, ::Sharded>, -) -> Q::Stored + key: C::Key, + lookup: QueryLookup<'_, CTX, C::Key, C::Sharded>, + query: &QueryVtable, +) -> C::Stored where - Q: QueryDescription, + C: QueryCache, + C::Key: Eq + Clone + Debug, + C::Stored: Clone, CTX: QueryContext, { - let job = match JobOwner::try_start(tcx, Q::query_state(tcx), span, &key, lookup, &Q::VTABLE) { + let job = match JobOwner::try_start(tcx, state, span, &key, lookup, query) { TryGetJob::NotYetStarted(job) => job, TryGetJob::Cycle(result) => return result, #[cfg(parallel_compiler)] @@ -406,18 +410,32 @@ where // expensive for some `DepKind`s. if !tcx.dep_graph().is_fully_enabled() { let null_dep_node = DepNode::new_no_params(DepKind::NULL); - return force_query_with_job(tcx, key, job, null_dep_node, &Q::VTABLE).0; + return force_query_with_job(tcx, key, job, null_dep_node, query).0; } - if Q::ANON { - let (result, dep_node_index) = try_execute_anon_query(tcx, key, job.id, &Q::VTABLE); + if query.anon { + let prof_timer = tcx.profiler().query_provider(); + + let ((result, dep_node_index), diagnostics) = with_diagnostics(|diagnostics| { + tcx.start_query(job.id, diagnostics, |tcx| { + tcx.dep_graph().with_anon_task(query.dep_kind, || query.compute(tcx, key)) + }) + }); + + prof_timer.finish_with_query_invocation_id(dep_node_index.into()); + + tcx.dep_graph().read_index(dep_node_index); + + if unlikely!(!diagnostics.is_empty()) { + tcx.store_diagnostics_for_anon_node(dep_node_index, diagnostics); + } return job.complete(tcx, result, dep_node_index); } - let dep_node = Q::to_dep_node(tcx, &key); + let dep_node = query.to_dep_node(tcx, &key); - if !Q::EVAL_ALWAYS { + if !query.eval_always { // The diagnostics for this query will be // promoted to the current session during // `try_mark_green()`, so we can ignore them here. @@ -431,7 +449,7 @@ where prev_dep_node_index, dep_node_index, &dep_node, - &Q::VTABLE, + query, ), dep_node_index, ) @@ -442,40 +460,11 @@ where } } - let (result, dep_node_index) = force_query_with_job(tcx, key, job, dep_node, &Q::VTABLE); + let (result, dep_node_index) = force_query_with_job(tcx, key, job, dep_node, query); tcx.dep_graph().read_index(dep_node_index); result } -fn try_execute_anon_query( - tcx: CTX, - key: K, - job_id: QueryJobId, - query: &QueryVtable, -) -> (V, DepNodeIndex) -where - CTX: QueryContext, -{ - debug_assert!(query.anon); - let prof_timer = tcx.profiler().query_provider(); - - let ((result, dep_node_index), diagnostics) = with_diagnostics(|diagnostics| { - tcx.start_query(job_id, diagnostics, |tcx| { - tcx.dep_graph().with_anon_task(query.dep_kind, || query.compute(tcx, key)) - }) - }); - - prof_timer.finish_with_query_invocation_id(dep_node_index.into()); - - tcx.dep_graph().read_index(dep_node_index); - - if unlikely!(!diagnostics.is_empty()) { - tcx.store_diagnostics_for_anon_node(dep_node_index, diagnostics); - } - - (result, dep_node_index) -} - fn load_from_disk_and_cache_in_memory( tcx: CTX, key: K, @@ -639,7 +628,7 @@ where tcx.dep_graph().read_index(index); value.clone() }, - |key, lookup| try_execute_query::(tcx, span, key, lookup), + |key, lookup| try_execute_query(tcx, Q::query_state(tcx), span, key, lookup, &Q::VTABLE), ) } From e15383cce12a3bf33ec0475fd78cf786f57c3140 Mon Sep 17 00:00:00 2001 From: Camille GILLOT Date: Sat, 28 Mar 2020 13:51:37 +0100 Subject: [PATCH 6/9] Move the DepNode construction to librustc_query_system. --- src/librustc_middle/dep_graph/dep_node.rs | 25 ++----------------- src/librustc_middle/dep_graph/mod.rs | 4 +++ src/librustc_middle/ty/query/mod.rs | 2 +- src/librustc_middle/ty/query/plumbing.rs | 6 ----- .../dep_graph/dep_node.rs | 24 ++++++++++++++++++ src/librustc_query_system/dep_graph/mod.rs | 1 + src/librustc_query_system/query/config.rs | 16 ++++++++---- src/librustc_query_system/query/plumbing.rs | 4 ++- 8 files changed, 46 insertions(+), 36 deletions(-) diff --git a/src/librustc_middle/dep_graph/dep_node.rs b/src/librustc_middle/dep_graph/dep_node.rs index f4a4aab844c14..3303790088010 100644 --- a/src/librustc_middle/dep_graph/dep_node.rs +++ b/src/librustc_middle/dep_graph/dep_node.rs @@ -183,31 +183,10 @@ macro_rules! define_dep_nodes { // tuple args $({ erase!($tuple_arg_ty); - let hash = DepNodeParams::to_fingerprint(&arg, _tcx); - let dep_node = DepNode { - kind: DepKind::$variant, - hash - }; - - #[cfg(debug_assertions)] - { - if !dep_node.kind.can_reconstruct_query_key() && - (_tcx.sess.opts.debugging_opts.incremental_info || - _tcx.sess.opts.debugging_opts.query_dep_graph) - { - _tcx.dep_graph.register_dep_node_debug_str(dep_node, || { - arg.to_debug_str(_tcx) - }); - } - } - - return dep_node; + return DepNode::construct(_tcx, DepKind::$variant, &arg) })* - DepNode { - kind: DepKind::$variant, - hash: Fingerprint::ZERO, - } + return DepNode::construct(_tcx, DepKind::$variant, &()) } )* } diff --git a/src/librustc_middle/dep_graph/mod.rs b/src/librustc_middle/dep_graph/mod.rs index 4786426792c3d..207c6d0fbff0f 100644 --- a/src/librustc_middle/dep_graph/mod.rs +++ b/src/librustc_middle/dep_graph/mod.rs @@ -98,6 +98,10 @@ impl<'tcx> DepContext for TyCtxt<'tcx> { fn debug_dep_tasks(&self) -> bool { self.sess.opts.debugging_opts.dep_tasks } + fn debug_dep_node(&self) -> bool { + self.sess.opts.debugging_opts.incremental_info + || self.sess.opts.debugging_opts.query_dep_graph + } fn try_force_from_dep_node(&self, dep_node: &DepNode) -> bool { // FIXME: This match is just a workaround for incremental bugs and should diff --git a/src/librustc_middle/ty/query/mod.rs b/src/librustc_middle/ty/query/mod.rs index 899479e65a732..105b0f8f2cf3a 100644 --- a/src/librustc_middle/ty/query/mod.rs +++ b/src/librustc_middle/ty/query/mod.rs @@ -1,4 +1,4 @@ -use crate::dep_graph::{self, DepConstructor, DepNode, DepNodeParams}; +use crate::dep_graph::{self, DepNode, DepNodeParams}; use crate::hir::exports::Export; use crate::hir::map; use crate::infer::canonical::{self, Canonical}; diff --git a/src/librustc_middle/ty/query/plumbing.rs b/src/librustc_middle/ty/query/plumbing.rs index e4237df592394..d6d4335e9388d 100644 --- a/src/librustc_middle/ty/query/plumbing.rs +++ b/src/librustc_middle/ty/query/plumbing.rs @@ -348,12 +348,6 @@ macro_rules! define_queries_inner { &tcx.queries.$name } - #[allow(unused)] - #[inline(always)] - fn to_dep_node(tcx: TyCtxt<$tcx>, key: &Self::Key) -> DepNode { - DepConstructor::$node(tcx, *key) - } - #[inline] fn compute(tcx: TyCtxt<'tcx>, key: Self::Key) -> Self::Value { let provider = tcx.queries.providers.get(key.query_crate()) diff --git a/src/librustc_query_system/dep_graph/dep_node.rs b/src/librustc_query_system/dep_graph/dep_node.rs index 99eb3cdc0b020..36343365ab6c4 100644 --- a/src/librustc_query_system/dep_graph/dep_node.rs +++ b/src/librustc_query_system/dep_graph/dep_node.rs @@ -64,6 +64,24 @@ impl DepNode { debug_assert!(!kind.has_params()); DepNode { kind, hash: Fingerprint::ZERO } } + + pub fn construct(tcx: Ctxt, kind: K, arg: &Key) -> DepNode + where + Ctxt: crate::query::QueryContext, + Key: DepNodeParams, + { + let hash = arg.to_fingerprint(tcx); + let dep_node = DepNode { kind, hash }; + + #[cfg(debug_assertions)] + { + if !kind.can_reconstruct_query_key() && tcx.debug_dep_node() { + tcx.dep_graph().register_dep_node_debug_str(dep_node, || arg.to_debug_str(tcx)); + } + } + + return dep_node; + } } impl fmt::Debug for DepNode { @@ -120,6 +138,12 @@ where } } +impl DepNodeParams for () { + fn to_fingerprint(&self, _: Ctxt) -> Fingerprint { + Fingerprint::ZERO + } +} + /// A "work product" corresponds to a `.o` (or other) file that we /// save in between runs. These IDs do not have a `DefId` but rather /// some independent path or string that persists between runs without diff --git a/src/librustc_query_system/dep_graph/mod.rs b/src/librustc_query_system/dep_graph/mod.rs index fbc91575ede41..f571e902211cd 100644 --- a/src/librustc_query_system/dep_graph/mod.rs +++ b/src/librustc_query_system/dep_graph/mod.rs @@ -28,6 +28,7 @@ pub trait DepContext: Copy { fn create_stable_hashing_context(&self) -> Self::StableHashingContext; fn debug_dep_tasks(&self) -> bool; + fn debug_dep_node(&self) -> bool; /// Try to force a dep node to execute and see if it's green. fn try_force_from_dep_node(&self, dep_node: &DepNode) -> bool; diff --git a/src/librustc_query_system/query/config.rs b/src/librustc_query_system/query/config.rs index a334e897e400e..f031b54346fa9 100644 --- a/src/librustc_query_system/query/config.rs +++ b/src/librustc_query_system/query/config.rs @@ -28,7 +28,6 @@ pub(crate) struct QueryVtable { pub anon: bool, pub dep_kind: CTX::DepKind, pub eval_always: bool, - pub to_dep_node: fn(CTX, &K) -> DepNode, // Don't use this method to compute query results, instead use the methods on TyCtxt pub compute: fn(CTX, K) -> V, @@ -40,8 +39,11 @@ pub(crate) struct QueryVtable { } impl QueryVtable { - pub(crate) fn to_dep_node(&self, tcx: CTX, key: &K) -> DepNode { - (self.to_dep_node)(tcx, key) + pub(crate) fn to_dep_node(&self, tcx: CTX, key: &K) -> DepNode + where + K: crate::dep_graph::DepNodeParams, + { + DepNode::construct(tcx, self.dep_kind, key) } pub(crate) fn compute(&self, tcx: CTX, key: K) -> V { @@ -79,7 +81,12 @@ pub trait QueryAccessors: QueryConfig { // Don't use this method to access query results, instead use the methods on TyCtxt fn query_state<'a>(tcx: CTX) -> &'a QueryState; - fn to_dep_node(tcx: CTX, key: &Self::Key) -> DepNode; + fn to_dep_node(tcx: CTX, key: &Self::Key) -> DepNode + where + Self::Key: crate::dep_graph::DepNodeParams, + { + DepNode::construct(tcx, Self::DEP_KIND, key) + } // Don't use this method to compute query results, instead use the methods on TyCtxt fn compute(tcx: CTX, key: Self::Key) -> Self::Value; @@ -117,7 +124,6 @@ where const VTABLE: QueryVtable = QueryVtable { anon: Q::ANON, dep_kind: Q::DEP_KIND, - to_dep_node: Q::to_dep_node, eval_always: Q::EVAL_ALWAYS, compute: Q::compute, hash_result: Q::hash_result, diff --git a/src/librustc_query_system/query/plumbing.rs b/src/librustc_query_system/query/plumbing.rs index 74a2288cfa790..d94f6bc300b2d 100644 --- a/src/librustc_query_system/query/plumbing.rs +++ b/src/librustc_query_system/query/plumbing.rs @@ -392,7 +392,7 @@ fn try_execute_query( ) -> C::Stored where C: QueryCache, - C::Key: Eq + Clone + Debug, + C::Key: Eq + Clone + Debug + crate::dep_graph::DepNodeParams, C::Stored: Clone, CTX: QueryContext, { @@ -616,6 +616,7 @@ where pub fn get_query(tcx: CTX, span: Span, key: Q::Key) -> Q::Stored where Q: QueryDescription, + Q::Key: crate::dep_graph::DepNodeParams, CTX: QueryContext, { debug!("ty::query::get_query<{}>(key={:?}, span={:?})", Q::NAME, key, span); @@ -642,6 +643,7 @@ where pub fn ensure_query(tcx: CTX, key: Q::Key) where Q: QueryDescription, + Q::Key: crate::dep_graph::DepNodeParams, CTX: QueryContext, { if Q::EVAL_ALWAYS { From 49e024ee7c404fb767d8560b1cf248929f8d7574 Mon Sep 17 00:00:00 2001 From: Camille GILLOT Date: Sat, 28 Mar 2020 14:09:53 +0100 Subject: [PATCH 7/9] Monomorphise the interface. --- src/librustc_query_system/query/plumbing.rs | 101 ++++++++++++++------ 1 file changed, 72 insertions(+), 29 deletions(-) diff --git a/src/librustc_query_system/query/plumbing.rs b/src/librustc_query_system/query/plumbing.rs index d94f6bc300b2d..c22ea24bb16bd 100644 --- a/src/librustc_query_system/query/plumbing.rs +++ b/src/librustc_query_system/query/plumbing.rs @@ -613,23 +613,28 @@ where } #[inline(never)] -pub fn get_query(tcx: CTX, span: Span, key: Q::Key) -> Q::Stored +fn get_query_impl( + tcx: CTX, + state: &QueryState, + span: Span, + key: C::Key, + query: &QueryVtable, +) -> C::Stored where - Q: QueryDescription, - Q::Key: crate::dep_graph::DepNodeParams, CTX: QueryContext, + C: QueryCache, + C::Key: Eq + Clone + crate::dep_graph::DepNodeParams, + C::Stored: Clone, { - debug!("ty::query::get_query<{}>(key={:?}, span={:?})", Q::NAME, key, span); - try_get_cached( tcx, - Q::query_state(tcx), + state, key, |value, index| { tcx.dep_graph().read_index(index); value.clone() }, - |key, lookup| try_execute_query(tcx, Q::query_state(tcx), span, key, lookup, &Q::VTABLE), + |key, lookup| try_execute_query(tcx, state, span, key, lookup, query), ) } @@ -640,21 +645,25 @@ where /// side-effects -- e.g., in order to report errors for erroneous programs. /// /// Note: The optimization is only available during incr. comp. -pub fn ensure_query(tcx: CTX, key: Q::Key) -where - Q: QueryDescription, - Q::Key: crate::dep_graph::DepNodeParams, +fn ensure_query_impl( + tcx: CTX, + state: &QueryState, + key: C::Key, + query: &QueryVtable, +) where + C: QueryCache, + C::Key: Eq + Clone + crate::dep_graph::DepNodeParams, CTX: QueryContext, { - if Q::EVAL_ALWAYS { - let _ = get_query::(tcx, DUMMY_SP, key); + if query.eval_always { + let _ = get_query_impl(tcx, state, DUMMY_SP, key, query); return; } // Ensuring an anonymous query makes no sense - assert!(!Q::ANON); + assert!(!query.anon); - let dep_node = Q::to_dep_node(tcx, &key); + let dep_node = query.to_dep_node(tcx, &key); match tcx.dep_graph().try_mark_green_and_read(tcx, &dep_node) { None => { @@ -664,7 +673,7 @@ where // DepNodeIndex. We must invoke the query itself. The performance cost // this introduces should be negligible as we'll immediately hit the // in-memory cache, or another query down the line will. - let _ = get_query::(tcx, DUMMY_SP, key); + let _ = get_query_impl(tcx, state, DUMMY_SP, key, query); } Some((_, dep_node_index)) => { tcx.profiler().query_cache_hit(dep_node_index.into()); @@ -672,9 +681,16 @@ where } } -pub fn force_query(tcx: CTX, key: Q::Key, span: Span, dep_node: DepNode) -where - Q: QueryDescription, +fn force_query_impl( + tcx: CTX, + state: &QueryState, + key: C::Key, + span: Span, + dep_node: DepNode, + query: &QueryVtable, +) where + C: QueryCache, + C::Key: Eq + Clone + crate::dep_graph::DepNodeParams, CTX: QueryContext, { // We may be concurrently trying both execute and force a query. @@ -682,21 +698,48 @@ where try_get_cached( tcx, - Q::query_state(tcx), + state, key, |_, _| { // Cache hit, do nothing }, |key, lookup| { - let job = - match JobOwner::try_start(tcx, Q::query_state(tcx), span, &key, lookup, &Q::VTABLE) - { - TryGetJob::NotYetStarted(job) => job, - TryGetJob::Cycle(_) => return, - #[cfg(parallel_compiler)] - TryGetJob::JobCompleted(_) => return, - }; - force_query_with_job(tcx, key, job, dep_node, &Q::VTABLE); + let job = match JobOwner::try_start(tcx, state, span, &key, lookup, query) { + TryGetJob::NotYetStarted(job) => job, + TryGetJob::Cycle(_) => return, + #[cfg(parallel_compiler)] + TryGetJob::JobCompleted(_) => return, + }; + force_query_with_job(tcx, key, job, dep_node, query); }, ); } + +pub fn get_query(tcx: CTX, span: Span, key: Q::Key) -> Q::Stored +where + Q: QueryDescription, + Q::Key: crate::dep_graph::DepNodeParams, + CTX: QueryContext, +{ + debug!("ty::query::get_query<{}>(key={:?}, span={:?})", Q::NAME, key, span); + + get_query_impl(tcx, Q::query_state(tcx), span, key, &Q::VTABLE) +} + +pub fn ensure_query(tcx: CTX, key: Q::Key) +where + Q: QueryDescription, + Q::Key: crate::dep_graph::DepNodeParams, + CTX: QueryContext, +{ + ensure_query_impl(tcx, Q::query_state(tcx), key, &Q::VTABLE) +} + +pub fn force_query(tcx: CTX, key: Q::Key, span: Span, dep_node: DepNode) +where + Q: QueryDescription, + Q::Key: crate::dep_graph::DepNodeParams, + CTX: QueryContext, +{ + force_query_impl(tcx, Q::query_state(tcx), key, span, dep_node, &Q::VTABLE) +} From 282d72f6bb6ad8c00b9d5bbd06e7946a37d412ef Mon Sep 17 00:00:00 2001 From: Camille GILLOT Date: Sun, 29 Mar 2020 11:44:40 +0200 Subject: [PATCH 8/9] Inline a few things. --- src/librustc_query_system/query/plumbing.rs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/librustc_query_system/query/plumbing.rs b/src/librustc_query_system/query/plumbing.rs index c22ea24bb16bd..bea064ab94c79 100644 --- a/src/librustc_query_system/query/plumbing.rs +++ b/src/librustc_query_system/query/plumbing.rs @@ -80,6 +80,7 @@ enum QueryResult { } impl QueryState { + #[inline(always)] pub fn iter_results( &self, f: impl for<'a> FnOnce( @@ -89,6 +90,7 @@ impl QueryState { self.cache.iter(&self.shards, |shard| &mut shard.cache, f) } + #[inline(always)] pub fn all_inactive(&self) -> bool { let shards = self.shards.lock_shards(); shards.iter().all(|shard| shard.active.is_empty()) @@ -645,6 +647,7 @@ where /// side-effects -- e.g., in order to report errors for erroneous programs. /// /// Note: The optimization is only available during incr. comp. +#[inline(never)] fn ensure_query_impl( tcx: CTX, state: &QueryState, @@ -681,7 +684,8 @@ fn ensure_query_impl( } } -fn force_query_impl( +#[inline(never)] +fn force_query_impl( tcx: CTX, state: &QueryState, key: C::Key, @@ -715,6 +719,7 @@ fn force_query_impl( ); } +#[inline(always)] pub fn get_query(tcx: CTX, span: Span, key: Q::Key) -> Q::Stored where Q: QueryDescription, @@ -726,6 +731,7 @@ where get_query_impl(tcx, Q::query_state(tcx), span, key, &Q::VTABLE) } +#[inline(always)] pub fn ensure_query(tcx: CTX, key: Q::Key) where Q: QueryDescription, @@ -735,6 +741,7 @@ where ensure_query_impl(tcx, Q::query_state(tcx), key, &Q::VTABLE) } +#[inline(always)] pub fn force_query(tcx: CTX, key: Q::Key, span: Span, dep_node: DepNode) where Q: QueryDescription, From e4976d0caf6a4b9d9d3ec8979427c6e744c8c38d Mon Sep 17 00:00:00 2001 From: Camille GILLOT Date: Mon, 30 Mar 2020 17:27:14 +0200 Subject: [PATCH 9/9] Restrict access. --- src/librustc_query_system/query/plumbing.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/librustc_query_system/query/plumbing.rs b/src/librustc_query_system/query/plumbing.rs index bea064ab94c79..cc7d0a1570355 100644 --- a/src/librustc_query_system/query/plumbing.rs +++ b/src/librustc_query_system/query/plumbing.rs @@ -29,7 +29,7 @@ use std::ptr; #[cfg(debug_assertions)] use std::sync::atomic::{AtomicUsize, Ordering}; -pub struct QueryStateShard { +pub(super) struct QueryStateShard { pub(super) cache: C, active: FxHashMap>,