-
Notifications
You must be signed in to change notification settings - Fork 279
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Summary: ## This stack Continues the implementation of the primitives that will validate submodule expansion in large repo's. bonsais. The goal of validation is to make sure that **the submodule metadata file is always kept in sync with the expansion**. How this will be done is more complicated than it seems, but on a high level we will 1 Get the fsnode in the submodule repo from the commit in the metadata file 2. Derive the fsnode for the new commit 3. Get the entry for the expansion and compare with the fsnode from step #1. 4. We'll perform many validations of consistency in this process (e.g. if expansion is changed, metadata file has to be changed as well). ## Challenge of validation To efficiently verify that the working copies match, we want to get fsnodes of both the commit being expanded in the submodule (easy) and the new commit being synced to the large repo. The second one is a problem, because in order to get it, we would need access to the large repo's blobstore and to derive the fsnode using the large repo's derived data. The problem with is that the primitives that rewrite commits (thus expand the submodules) are, on purpose, only given the target repo's id, instead of the actual repo (and its blobstore). This was done a while back (D43002436) to make sure we don't accidentally write the target repo. We want to keep the maintain that safety net, so the problem this diff tries to solve is: **provide READ-ONLY access to some parts of the large repo to the commit sync primitives** (e.g. CommitInMemorySyncer). ## This diff Introduces a new Repo facet container, called `InMemoryRepo`, which can be passed to the validation primitives and will ensure that any writes performed during validation only happen in memory, i.e. are not persisted to the large repo's blobstore or tables. This InMemoryRepo needs to have `RepoDerivedData` and `Changesets`. But to create a `RepoDerivedData`, I need to provide a bunch of other things, most of which I won't need. So instead of implementing a "wrapper" for all of these things (e..g `BonsaiGitMapping`, `BonsaiHgMapping`), I defined a dummy data type that will implement these to satisfy the type checker. This is good because it will be very clear what the validation primitives will be using and if in the future this changes, the tests will crash because they'll try to read from or write to storage that they're not supposed to. In which case the person making the change will implement a wrapper to ensure the operations only write to memory. Reviewed By: markbt Differential Revision: D55701183 fbshipit-source-id: 706b8d4cbef8cde6a9562a52009db90901d4a685
- Loading branch information
1 parent
835a118
commit 54b6cee
Showing
10 changed files
with
554 additions
and
3 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
197 changes: 197 additions & 0 deletions
197
eden/mononoke/commit_rewriting/cross_repo_sync/src/git_submodules/dummy_struct.rs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,197 @@ | ||
| /* | ||
| * Copyright (c) Meta Platforms, Inc. and affiliates. | ||
| * | ||
| * This software may be used and distributed according to the terms of the | ||
| * GNU General Public License version 2. | ||
| */ | ||
|
|
||
| use std::collections::HashMap; | ||
|
|
||
| use ::sql::Transaction; | ||
| use anyhow::Error; | ||
| use anyhow::Result; | ||
| use async_trait::async_trait; | ||
| use bonsai_git_mapping::AddGitMappingErrorKind; | ||
| use bonsai_git_mapping::BonsaiGitMapping; | ||
| use bonsai_git_mapping::BonsaiGitMappingEntry; | ||
| use bonsai_git_mapping::BonsaisOrGitShas; | ||
| use commit_graph_types::edges::ChangesetEdges; | ||
| use commit_graph_types::storage::CommitGraphStorage; | ||
| use commit_graph_types::storage::FetchedChangesetEdges; | ||
| use commit_graph_types::storage::Prefetch; | ||
| use context::CoreContext; | ||
| use filenodes::FilenodeInfo; | ||
| use filenodes::FilenodeRange; | ||
| use filenodes::FilenodeResult; | ||
| use filenodes::Filenodes; | ||
| use filenodes::PreparedFilenode; | ||
| use mercurial_types::HgFileNodeId; | ||
| use mononoke_types::hash::GitSha1; | ||
| use mononoke_types::ChangesetId; | ||
| use mononoke_types::ChangesetIdPrefix; | ||
| use mononoke_types::ChangesetIdsResolvedFromPrefix; | ||
| use mononoke_types::RepoPath; | ||
| use mononoke_types::RepositoryId; | ||
| use vec1::Vec1; | ||
|
|
||
| /// Struct created to satisfy the type system when creating a `RepoDerivedData` | ||
| /// for the `InMemoryRepo`. | ||
| pub(crate) struct DummyStruct; | ||
|
|
||
| #[async_trait] | ||
| impl Filenodes for DummyStruct { | ||
| async fn add_filenodes( | ||
| &self, | ||
| _ctx: &CoreContext, | ||
| _info: Vec<PreparedFilenode>, | ||
| ) -> Result<FilenodeResult<()>> { | ||
| unimplemented!() | ||
| } | ||
|
|
||
| async fn add_or_replace_filenodes( | ||
| &self, | ||
| _ctx: &CoreContext, | ||
| _info: Vec<PreparedFilenode>, | ||
| ) -> Result<FilenodeResult<()>> { | ||
| unimplemented!() | ||
| } | ||
|
|
||
| async fn get_filenode( | ||
| &self, | ||
| _ctx: &CoreContext, | ||
| _path: &RepoPath, | ||
| _filenode: HgFileNodeId, | ||
| ) -> Result<FilenodeResult<Option<FilenodeInfo>>> { | ||
| unimplemented!() | ||
| } | ||
|
|
||
| async fn get_all_filenodes_maybe_stale( | ||
| &self, | ||
| _ctx: &CoreContext, | ||
| _path: &RepoPath, | ||
| _limit: Option<u64>, | ||
| ) -> Result<FilenodeResult<FilenodeRange>> { | ||
| unimplemented!() | ||
| } | ||
|
|
||
| fn prime_cache(&self, _ctx: &CoreContext, _filenodes: &[PreparedFilenode]) { | ||
| unimplemented!() | ||
| } | ||
| } | ||
|
|
||
| #[async_trait] | ||
| impl BonsaiGitMapping for DummyStruct { | ||
| fn repo_id(&self) -> RepositoryId { | ||
| unimplemented!() | ||
| } | ||
|
|
||
| async fn add( | ||
| &self, | ||
| _ctx: &CoreContext, | ||
| _entry: BonsaiGitMappingEntry, | ||
| ) -> Result<(), AddGitMappingErrorKind> { | ||
| unimplemented!() | ||
| } | ||
|
|
||
| async fn bulk_add( | ||
| &self, | ||
| _ctx: &CoreContext, | ||
| _entries: &[BonsaiGitMappingEntry], | ||
| ) -> Result<(), AddGitMappingErrorKind> { | ||
| unimplemented!() | ||
| } | ||
|
|
||
| async fn bulk_add_git_mapping_in_transaction( | ||
| &self, | ||
| _ctx: &CoreContext, | ||
| _entries: &[BonsaiGitMappingEntry], | ||
| _transaction: Transaction, | ||
| ) -> Result<Transaction, AddGitMappingErrorKind> { | ||
| unimplemented!() | ||
| } | ||
|
|
||
| async fn get( | ||
| &self, | ||
| _ctx: &CoreContext, | ||
| _cs: BonsaisOrGitShas, | ||
| ) -> Result<Vec<BonsaiGitMappingEntry>, Error> { | ||
| unimplemented!() | ||
| } | ||
|
|
||
| /// Use caching for the ranges of one element, use slower path otherwise. | ||
| async fn get_in_range( | ||
| &self, | ||
| _ctx: &CoreContext, | ||
| _low: GitSha1, | ||
| _high: GitSha1, | ||
| _limit: usize, | ||
| ) -> Result<Vec<GitSha1>, Error> { | ||
| unimplemented!() | ||
| } | ||
| } | ||
|
|
||
| #[async_trait] | ||
| impl CommitGraphStorage for DummyStruct { | ||
| fn repo_id(&self) -> RepositoryId { | ||
| unimplemented!() | ||
| } | ||
|
|
||
| async fn add(&self, _ctx: &CoreContext, _edges: ChangesetEdges) -> Result<bool> { | ||
| unimplemented!() | ||
| } | ||
|
|
||
| async fn add_many( | ||
| &self, | ||
| _ctx: &CoreContext, | ||
| _many_edges: Vec1<ChangesetEdges>, | ||
| ) -> Result<usize> { | ||
| unimplemented!() | ||
| } | ||
|
|
||
| async fn fetch_edges(&self, _ctx: &CoreContext, _cs_id: ChangesetId) -> Result<ChangesetEdges> { | ||
| unimplemented!() | ||
| } | ||
|
|
||
| async fn maybe_fetch_edges( | ||
| &self, | ||
| _ctx: &CoreContext, | ||
| _cs_id: ChangesetId, | ||
| ) -> Result<Option<ChangesetEdges>> { | ||
| unimplemented!() | ||
| } | ||
|
|
||
| async fn fetch_many_edges( | ||
| &self, | ||
| _ctx: &CoreContext, | ||
| _cs_ids: &[ChangesetId], | ||
| _prefetch: Prefetch, | ||
| ) -> Result<HashMap<ChangesetId, FetchedChangesetEdges>> { | ||
| unimplemented!() | ||
| } | ||
|
|
||
| async fn maybe_fetch_many_edges( | ||
| &self, | ||
| _ctx: &CoreContext, | ||
| _cs_ids: &[ChangesetId], | ||
| _prefetch: Prefetch, | ||
| ) -> Result<HashMap<ChangesetId, FetchedChangesetEdges>> { | ||
| unimplemented!() | ||
| } | ||
|
|
||
| async fn find_by_prefix( | ||
| &self, | ||
| _ctx: &CoreContext, | ||
| _cs_prefix: ChangesetIdPrefix, | ||
| _limit: usize, | ||
| ) -> Result<ChangesetIdsResolvedFromPrefix> { | ||
| unimplemented!() | ||
| } | ||
|
|
||
| async fn fetch_children( | ||
| &self, | ||
| _ctx: &CoreContext, | ||
| _cs_id: ChangesetId, | ||
| ) -> Result<Vec<ChangesetId>> { | ||
| unimplemented!() | ||
| } | ||
| } |
Oops, something went wrong.