8000 refactor: lazy git by NikolaMilosa · Pull Request #837 · dfinity/dre · GitHub
[go: up one dir, main page]
More Web Proxy on the site http://driver.im/
Skip to content

refactor: lazy git #837

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 7 commits into from
Sep 2, 2024
Merged

refactor: lazy git #837

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
2a01701
renaming lazy git
NikolaMilosa Aug 31, 2024
8eca9f3
refactoring lazy repo
NikolaMilosa Aug 31, 2024
34204cd
using dyn lazy git
NikolaMilosa Aug 31, 2024
b56d26b
Branch was auto-updated.
sa-github-api Sep 2, 2024
bbc5049
Branch was auto-updated.
sa-github-api Sep 2, 2024
63b4b3b
Branch was auto-updated.
sa-github-api Sep 2, 2024
19f2f60
Branch was auto-updated.
sa-github-api Sep 2, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions rs/cli/src/ctx.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ use std::{

use ic_canisters::{governance::governance_canister_version, CanisterClient, IcAgentCanisterClient};
use ic_management_backend::{
lazy_git::LazyGit,
lazy_registry::{LazyRegistry, LazyRegistryImpl},
proposal::ProposalAgent,
registry::{local_registry_path, sync_local_store},
Expand All @@ -32,6 +33,7 @@ pub struct DreContext {
registry: RefCell<Option<Arc<dyn LazyRegistry>>>,
ic_admin: Option<Arc<IcAdminWrapper>>,
runner: RefCell<Option<Rc<Runner>>>,
ic_repo: RefCell<Option<Arc<dyn LazyGit>>>,
verbose_runner: bool,
skip_sync: bool,
ic_admin_path: Option<String>,
Expand Down Expand Up @@ -85,6 +87,7 @@ impl DreContext {
skip_sync: args.no_sync,
ic_admin_path,
forum_post_link: args.forum_post_link.clone(),
ic_repo: RefCell::new(None),
})
}

Expand Down Expand Up @@ -232,6 +235,7 @@ impl DreContext {
self.network().clone(),
self.proposals_agent(),
self.verbose_runner,
self.ic_repo.clone(),
));
*self.runner.borrow_mut() = Some(runner.clone());
runner
Expand Down
23 changes: 15 additions & 8 deletions rs/cli/src/runner.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
use std::cell::RefCell;
use std::collections::BTreeMap;
use std::collections::BTreeSet;
use std::rc::Rc;
use std::sync::Arc;

use decentralization::network::DecentralizedSubnet;
Expand All @@ -16,6 +15,7 @@ use futures_util::future::try_join;
use ic_management_backend::health;
use ic_management_backend::health::HealthStatusQuerier;
use ic_management_backend::lazy_git::LazyGit;
use ic_management_backend::lazy_git::LazyGitImpl;
use ic_management_backend::lazy_registry::LazyRegistry;
use ic_management_backend::proposal::ProposalAgent;
use ic_management_backend::registry::ReleasesOps;
Expand Down Expand Up @@ -45,31 +45,38 @@ use crate::operations::hostos_rollout::NodeGroupUpdate;
pub struct Runner {
ic_admin: Arc<IcAdminWrapper>,
registry: Arc<dyn LazyRegistry>,
ic_repo: RefCell<Option<Rc<LazyGit>>>,
ic_repo: RefCell<Option<Arc<dyn LazyGit>>>,
network: Network,
proposal_agent: ProposalAgent,
verbose: bool,
}

impl Runner {
pub fn new(ic_admin: Arc<IcAdminWrapper>, registry: Arc<dyn LazyRegistry>, network: Network, agent: ProposalAgent, verbose: bool) -> Self {
pub fn new(
ic_admin: Arc<IcAdminWrapper>,
registry: Arc<dyn LazyRegistry>,
network: Network,
agent: ProposalAgent,
verbose: bool,
ic_repo: RefCell<Option<Arc<dyn LazyGit>>>,
) -> Self {
Self {
ic_admin,
registry,
ic_repo: RefCell::new(None),
ic_repo,
network,
proposal_agent: agent,
verbose,
}
}

async fn ic_repo(&self) -> Rc<LazyGit> {
async fn ic_repo(&self) -> Arc<dyn LazyGit> {
if let Some(ic_repo) = self.ic_repo.borrow().as_ref() {
return ic_repo.clone();
}

let ic_repo = Rc::new(
LazyGit::new(
let ic_repo = Arc::new(
LazyGitImpl::new(
self.network.clone(),
self.registry
.elected_guestos()
Expand All @@ -83,7 +90,7 @@ impl Runner {
.to_vec(),
)
.expect("Should be able to create IC repo"),
);
) as Arc<dyn LazyGit>;
*self.ic_repo.borrow_mut() = Some(ic_repo.clone());
ic_repo
}
Expand Down
95 changes: 55 additions & 40 deletions rs/ic-management-backend/src/lazy_git.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,68 +1,83 @@
use std::{
cell::RefCell,
collections::{HashMap, HashSet},
sync::Arc,
};

use futures::future::BoxFuture;
use ic_management_types::{Artifact, ArtifactReleases, Network, Release};
use itertools::Itertools;
use lazy_static::lazy_static;
use log::{debug, error};
use regex::Regex;
use tokio::sync::RwLock;

use crate::git_ic_repo::IcRepo;

pub struct LazyGit {
guestos_releases: RefCell<Option<Arc<ArtifactReleases>>>,
hostos_releases: RefCell<Option<Arc<ArtifactReleases>>>,
ic_repo: RefCell<IcRepo>,
pub trait LazyGit: Send + Sync {
fn guestos_releases(&self) -> BoxFuture<'_, anyhow::Result<Arc<ArtifactReleases>>>;

fn hostos_releases(&self) -> BoxFuture<'_, anyhow::Result<Arc<ArtifactReleases>>>;
}

pub struct LazyGitImpl {
guestos_releases: RwLock<Option<Arc<ArtifactReleases>>>,
hostos_releases: RwLock<Option<Arc<ArtifactReleases>>>,
ic_repo: RwLock<IcRepo>,
network: Network,
blessed_replica_versions: Vec<String>,
elected_hostos_versions: Vec<String>,
}

impl LazyGit {
pub fn new(network: Network, blessed_replica_versions: Vec<String>, elected_hostos_versions: Vec<String>) -> anyhow::Result<Self> {
Ok(Self {
guestos_releases: RefCell::new(None),
hostos_releases: RefCell::new(None),
ic_repo: RefCell::new(IcRepo::new()?),
network,
blessed_replica_versions,
elected_hostos_versions,
impl LazyGit for LazyGitImpl {
fn guestos_releases(&self) -> BoxFuture<'_, anyhow::Result<Arc<ArtifactReleases>>> {
Box::pin(async {
if let Some(releases) = self.guestos_releases.read().await.as_ref() {
return Ok(releases.to_owned());
}

self.update_releases().await?;
self.guestos_releases
.read()
.await
.as_ref()
.map(|n| n.to_owned())
.ok_or(anyhow::anyhow!("Failed to update releases"))
})
}

pub async fn guestos_releases(&self) -> anyhow::Result<Arc<ArtifactReleases>> {
if let Some(releases) = self.guestos_releases.borrow().as_ref() {
return Ok(releases.to_owned());
}
fn hostos_releases(&self) -> BoxFuture<'_, anyhow::Result<Arc<ArtifactReleases>>> {
Box::pin(async {
if let Some(releases) = self.hostos_releases.read().await.as_ref() {
return Ok(releases.to_owned());
}

self.update_releases().await?;
self.guestos_releases
.borrow()
.as_ref()
.map(|n| n.to_owned())
.ok_or(anyhow::anyhow!("Failed to update releases"))
self.update_releases().await?;
self.hostos_releases
.read()
.await
.as_ref()
.map(|n| n.to_owned())
.ok_or(anyhow::anyhow!("Failed to update releases"))
})
}
}

pub async fn hostos_releases(&self) -> anyhow::Result<Arc<ArtifactReleases>> {
if let Some(releases) = self.hostos_releases.borrow().as_ref() {
return Ok(releases.to_owned());
}

self.update_releases().await?;
self.hostos_releases
.borrow()
.as_ref()
.map(|n| n.to_owned())
.ok_or(anyhow::anyhow!("Failed to update releases"))
impl LazyGitImpl {
pub fn new(network: Network, blessed_replica_versions: Vec<String>, elected_hostos_versions: Vec<String>) -> anyhow::Result<Self> {
Ok(Self {
guestos_releases: RwLock::new(None),
hostos_releases: RwLock::new(None),
ic_repo: RwLock::new(IcRepo::new()?),< 6DAF /span>
network,
blessed_replica_versions,
elected_hostos_versions,
})
}

async fn update_releases(&self) -> anyhow::Result<()> {
if !self.network.eq(&Network::mainnet_unchecked()?) {
*self.guestos_releases.borrow_mut() = Some(Arc::new(ArtifactReleases::new(Artifact::GuestOs)));
*self.hostos_releases.borrow_mut() = Some(Arc::new(ArtifactReleases::new(Artifact::HostOs)));
*self.guestos_releases.write().await = Some(Arc::new(ArtifactReleases::new(Artifact::GuestOs)));
*self.hostos_releases.write().await = Some(Arc::new(ArtifactReleases::new(Artifact::HostOs)));
return Ok(());
}

Expand All @@ -84,8 +99,8 @@ impl LazyGit {
// A HashMap from the git revision to the latest commit branch in which the
// commit is present
let mut commit_to_release: HashMap<String, Release> = HashMap::new();
let mut ic_repo = self.ic_repo.write().await;
blessed_versions.into_iter().for_each(|commit_hash| {
let mut ic_repo = self.ic_repo.borrow_mut();
match ic_repo.get_branches_with_commit(commit_hash) {
// For each commit get a list of branches that have the commit
Ok(branches) => {
Expand Down Expand Up @@ -126,8 +141,8 @@ impl LazyGit {
});

for (blessed_versions, mut to_update, artifact_type) in [
(&self.blessed_replica_versions, self.guestos_releases.borrow_mut(), Artifact::GuestOs),
(&self.elected_hostos_versions, self.hostos_releases.borrow_mut(), Artifact::HostOs),
(&self.blessed_replica_versions, self.guestos_releases.write().await, Artifact::GuestOs),
(&self.elected_hostos_versions, self.hostos_releases.write().await, Artifact::HostOs),
] {
let releases = blessed_versions
.iter()
Expand Down
Loading
0