Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions crates/openshell-server/src/compute/mutation_guard.rs
Original file line number Diff line number Diff line change
Expand Up @@ -478,6 +478,12 @@ impl ComputeRuntime {
pub(crate) fn set_mutation_lock_timeout_for_tests(&self, timeout: Duration) {
self.mutation_locks.set_timeout_for_tests(timeout);
}

#[cfg(test)]
pub(crate) fn sandbox_mutation_lock_references_for_tests(&self, sandbox_id: &str) -> usize {
self.mutation_locks
.key_references(MutationLockKey::Sandbox(sandbox_id).advisory_key())
}
}

#[cfg(test)]
Expand Down
124 changes: 123 additions & 1 deletion crates/openshell-server/src/grpc/policy.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2711,6 +2711,7 @@ async fn persist_existing_policy_projection(
annotations: &HashMap<String, String>,
current_annotations: &HashMap<String, String>,
backfill_policy: Option<&ProtoSandboxPolicy>,
replace_startup_baseline: bool,
) -> Result<HashMap<String, String>, Status> {
let annotations_match = annotations
.iter()
Expand All @@ -2728,7 +2729,7 @@ async fn persist_existing_policy_projection(
crate::policy_store::permits_initial_static_policy_repair(sandbox);
if let Some(policy) = backfill_policy.as_ref()
&& let Some(spec) = sandbox.spec.as_mut()
&& (spec.policy.is_none() || startup_blocked)
&& (spec.policy.is_none() || (startup_blocked && replace_startup_baseline))
{
spec.policy = Some(policy.clone());
}
Expand Down Expand Up @@ -4618,6 +4619,7 @@ async fn handle_update_config_inner(
&req.annotations,
&response_annotations,
None,
false,
)
.await?
};
Expand Down Expand Up @@ -4768,6 +4770,7 @@ async fn handle_update_config_inner(
&req.annotations,
&response_annotations,
backfill_policy.as_ref(),
startup_blocked && !sandbox_caller,
)
.await?;
if backfill_policy.is_some() {
Expand Down Expand Up @@ -4798,6 +4801,7 @@ async fn handle_update_config_inner(
expected_resource_version: req.expected_resource_version,
annotations: req.annotations.clone(),
backfill_policy: backfill_policy.clone(),
replace_startup_baseline: startup_blocked && !sandbox_caller,
};

match state.store.put_policy_revision_atomic(&write).await {
Expand Down Expand Up @@ -7737,6 +7741,7 @@ async fn apply_merge_operations_with_retry(
expected_resource_version: context.expected_resource_version,
annotations: context.annotations.clone(),
backfill_policy: None,
replace_startup_baseline: false,
})
.await
.map(Some)
Expand Down Expand Up @@ -8980,6 +8985,123 @@ mod tests {
assert!(!diagnostic.contains('\n'));
}

#[tokio::test]
async fn startup_policy_backfill_preserves_repair_while_waiting_for_mutation_lock() {
use openshell_core::proto::{
ConfigurationAdmissionState as Admission, SandboxConfigurationAdmission, SandboxPhase,
};
use std::time::Duration;

for admission in [Admission::Pending, Admission::Rejected] {
let state = test_server_state().await;
let sandbox_id = "sb-startup-backfill-race";
let sandbox_name = "startup-backfill-race";
let image_policy = test_policy_with_rule("image", "image.example.com");
let repaired_policy = test_policy_with_rule("repair", "repaired.example.com");
let mut sandbox =
test_sandbox(sandbox_id, sandbox_name, image_policy.clone(), Vec::new());
sandbox.spec.as_mut().unwrap().policy = None;
sandbox.set_phase(SandboxPhase::Provisioning.into());
let status = sandbox.status.as_mut().unwrap();
status.configuration_activated = Some(false);
status.configuration_admission = Some(SandboxConfigurationAdmission {
state: admission.into(),
..Default::default()
});
state.store.put_message(&sandbox).await.unwrap();

let update = |policy| UpdateConfigRequest {
sandbox: sandbox_name.to_string(),
workspace_scope: Some(openshell_core::proto::workspace_selector("default")),
policy: Some(policy),
..Default::default()
};
let repair_request = with_user(Request::new(update(repaired_policy.clone())));
let image_request = with_sandbox(Request::new(update(image_policy)), sandbox_id);
let fresh_request =
with_sandbox(Request::new(update(repaired_policy.clone())), sandbox_id);
let held = crate::persistence::lock_order::branch(
state
.compute
.mutation_guard(MutationScope::sandbox("default", sandbox_id)),
)
.await
.unwrap();

let repair_state = state.clone();
let repair =
tokio::spawn(
async move { handle_update_config(&repair_state, repair_request).await },
);
tokio::time::timeout(Duration::from_secs(5), async {
while state
.compute
.sandbox_mutation_lock_references_for_tests(sandbox_id)
< 2
{
tokio::task::yield_now().await;
}
})
.await
.expect("management repair is first in the lock queue");

let image_state = state.clone();
let image =
tokio::spawn(
async move { handle_update_config(&image_state, image_request).await },
);
tokio::time::timeout(Duration::from_secs(5), async {
while state
.compute
.sandbox_mutation_lock_references_for_tests(sandbox_id)
< 3
{
tokio::task::yield_now().await;
}
})
.await
.expect("image upload read the empty baseline and queued behind the repair");
drop(held);

let repaired = repair.await.unwrap().unwrap().into_inner();
let error = image
.await
.unwrap()
.expect_err("sandbox backfill must not replace the management baseline");
assert_eq!(error.code(), Code::Aborted);
let saved = state
.store
.get_message::<Sandbox>(sandbox_id)
.await
.unwrap()
.unwrap();
assert_eq!(saved.spec.unwrap().policy, Some(repaired_policy));
let latest = state
.store
.get_latest_policy(sandbox_id)
.await
.unwrap()
.unwrap();
assert_eq!(latest.version, i64::from(repaired.version));
assert_eq!(latest.policy_hash, repaired.policy_hash);
assert_eq!(
state
.store
.list_policies(sandbox_id, 10, 0)
.await
.unwrap()
.len(),
1,
"rejected backfill must not leave an orphan policy revision"
);
let fresh = handle_update_config(&state, fresh_request)
.await
.expect("a fresh sync can use the repaired policy")
.into_inner();
assert_eq!(fresh.version, repaired.version);
}
}

#[tokio::test]
async fn configuration_admission_retains_invalid_image_composition_for_repair() {
use openshell_core::proto::{
Expand Down
3 changes: 3 additions & 0 deletions crates/openshell-server/src/persistence/tests.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1530,6 +1530,7 @@ async fn policy_atomic_write_commits_revision_provenance_and_sandbox_projection(
expected_resource_version: current_version,
annotations: provenance.clone(),
backfill_policy: Some(policy.clone()),
replace_startup_baseline: false,
})
.await
.unwrap();
Expand Down Expand Up @@ -1599,6 +1600,7 @@ async fn policy_atomic_write_rolls_back_sandbox_when_revision_insert_conflicts()
expected_resource_version: before_version,
annotations: StdHashMap::from([("signature".to_string(), "new".to_string())]),
backfill_policy: Some(policy),
replace_startup_baseline: false,
})
.await
.unwrap_err();
Expand Down Expand Up @@ -1644,6 +1646,7 @@ async fn policy_atomic_write_persists_workspace() {
expected_resource_version: current_version,
annotations: StdHashMap::new(),
backfill_policy: Some(policy),
replace_startup_baseline: false,
})
.await
.unwrap();
Expand Down
31 changes: 29 additions & 2 deletions crates/openshell-server/src/policy_store.rs
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,10 @@ pub struct AtomicPolicyRevisionWrite {
/// Populate the create-time baseline, or replace it while startup admission
/// is blocked and no workload has consumed the static restrictions.
pub backfill_policy: Option<ProtoSandboxPolicy>,
/// Allow `backfill_policy` to replace an existing baseline while startup
/// admission is blocked. Only a management repair may do this; a
/// sandbox-originated write must conflict instead of overwriting it.
pub replace_startup_baseline: bool,
}

pub fn policy_record_for_atomic_write(
Expand Down Expand Up @@ -108,7 +112,7 @@ pub fn project_policy_revision_onto_sandbox(
changed = true;
}
Some(current) if current == backfill_policy => {}
Some(_) if startup_blocked => {
Some(_) if startup_blocked && write.replace_startup_baseline => {
spec.policy = Some(backfill_policy.clone());
changed = true;
}
Expand Down Expand Up @@ -729,7 +733,7 @@ mod tests {
.unwrap()
.read_write
.push("/new-static-path".to_string());
let write = AtomicPolicyRevisionWrite {
let mut write = AtomicPolicyRevisionWrite {
id: "revision".to_string(),
sandbox_id: "sandbox".to_string(),
workspace: "default".to_string(),
Expand All @@ -740,6 +744,7 @@ mod tests {
expected_resource_version: 0,
annotations: HashMap::new(),
backfill_policy: Some(replacement.clone()),
replace_startup_baseline: true,
};
for activated in [None, Some(false), Some(true)] {
for state in [Admission::Pending, Admission::Rejected, Admission::Accepted] {
Expand Down Expand Up @@ -772,5 +777,27 @@ mod tests {
}
}
}
// A sandbox backfill chosen from an empty baseline cannot replace a
// management repair that commits before its atomic projection.
write.replace_startup_baseline = false;
let blocked = Sandbox {
spec: Some(SandboxSpec {
policy: Some(baseline),
..Default::default()
}),
status: Some(SandboxStatus {
configuration_activated: Some(false),
configuration_admission: Some(SandboxConfigurationAdmission {
state: Admission::Pending.into(),
..Default::default()
}),
..Default::default()
}),
..Default::default()
};
assert!(matches!(
project_policy_revision_onto_sandbox(&write, &blocked.encode_to_vec(), 1),
Err(PersistenceError::Conflict { .. })
));
}
}
Loading
Loading