Compare commits
9 commits
d812bb51e2
...
8176a8261e
| Author | SHA1 | Date | |
|---|---|---|---|
| 8176a8261e | |||
| e221cd8502 | |||
| 421544786f | |||
| 23c3572106 | |||
| 9c6cb11713 | |||
| 17d5987517 | |||
| f353660f97 | |||
| 6cb11af642 | |||
| 368558d9d8 |
25 changed files with 2179 additions and 323 deletions
|
|
@ -40,6 +40,7 @@ This architecture provides compile-time correctness, observability through event
|
||||||
- Compile time correctness is a super-power, and investment in it speeds up flywheel for development and user value.
|
- Compile time correctness is a super-power, and investment in it speeds up flywheel for development and user value.
|
||||||
- **CLI/Service Interchangeability**: Both the CLI and service must produce identical artifacts (BEL events, logs, metrics, outputs) in the same locations. Users should be able to build with one interface and query/inspect results from the other seamlessly. This principle applies to all DataBuild operations, not just builds.
|
- **CLI/Service Interchangeability**: Both the CLI and service must produce identical artifacts (BEL events, logs, metrics, outputs) in the same locations. Users should be able to build with one interface and query/inspect results from the other seamlessly. This principle applies to all DataBuild operations, not just builds.
|
||||||
- The BEL represents real things that happen: job run processes that are started or fail, requests from the user, dep misses, etc.
|
- The BEL represents real things that happen: job run processes that are started or fail, requests from the user, dep misses, etc.
|
||||||
|
- We focus on highly impactful tests anchored to stable interfaces. For instance, using BEL events to create valid application states to test orchestration logic via shared scenarios. This helps us keep a high ratio from "well tested functionality" to "test brittleness".
|
||||||
|
|
||||||
## Build & Test
|
## Build & Test
|
||||||
```bash
|
```bash
|
||||||
|
|
|
||||||
|
|
@ -350,11 +350,13 @@ impl Clone for BuildEventLog<MemoryBELStorage> {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
mod sqlite_bel_storage {
|
mod sqlite_bel_storage {
|
||||||
use crate::build_event_log::{BELStorage, BuildEventLog, SqliteBELStorage};
|
use crate::build_event_log::{BELStorage, BuildEventLog, SqliteBELStorage};
|
||||||
use crate::build_state::BuildState;
|
use crate::build_state::BuildState;
|
||||||
use crate::data_build_event::Event;
|
use crate::data_build_event::Event;
|
||||||
|
use crate::util::test_scenarios::default_originating_lifetime;
|
||||||
use crate::{PartitionRef, WantCreateEventV1};
|
use crate::{PartitionRef, WantCreateEventV1};
|
||||||
use uuid::Uuid;
|
use uuid::Uuid;
|
||||||
|
|
||||||
|
|
@ -387,6 +389,7 @@ mod tests {
|
||||||
e.partitions = vec![PartitionRef {
|
e.partitions = vec![PartitionRef {
|
||||||
r#ref: "sqlite_partition_1234".to_string(),
|
r#ref: "sqlite_partition_1234".to_string(),
|
||||||
}];
|
}];
|
||||||
|
e.lifetime = Some(default_originating_lifetime());
|
||||||
let event_id = log
|
let event_id = log
|
||||||
.append_event(&Event::WantCreateV1(e))
|
.append_event(&Event::WantCreateV1(e))
|
||||||
.expect("append_event failed");
|
.expect("append_event failed");
|
||||||
|
|
@ -430,14 +433,17 @@ mod tests {
|
||||||
|
|
||||||
let mut e2 = WantCreateEventV1::default();
|
let mut e2 = WantCreateEventV1::default();
|
||||||
e2.want_id = Uuid::new_v4().into();
|
e2.want_id = Uuid::new_v4().into();
|
||||||
|
e2.lifetime = Some(default_originating_lifetime());
|
||||||
log.append_event(&Event::WantCreateV1(e2))
|
log.append_event(&Event::WantCreateV1(e2))
|
||||||
.expect("append_event failed");
|
.expect("append_event failed");
|
||||||
let mut e3 = WantCreateEventV1::default();
|
let mut e3 = WantCreateEventV1::default();
|
||||||
e3.want_id = Uuid::new_v4().into();
|
e3.want_id = Uuid::new_v4().into();
|
||||||
|
e3.lifetime = Some(default_originating_lifetime());
|
||||||
log.append_event(&Event::WantCreateV1(e3))
|
log.append_event(&Event::WantCreateV1(e3))
|
||||||
.expect("append_event failed");
|
.expect("append_event failed");
|
||||||
let mut e4 = WantCreateEventV1::default();
|
let mut e4 = WantCreateEventV1::default();
|
||||||
e4.want_id = Uuid::new_v4().into();
|
e4.want_id = Uuid::new_v4().into();
|
||||||
|
e4.lifetime = Some(default_originating_lifetime());
|
||||||
log.append_event(&Event::WantCreateV1(e4))
|
log.append_event(&Event::WantCreateV1(e4))
|
||||||
.expect("append_event failed");
|
.expect("append_event failed");
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -4,12 +4,12 @@
|
||||||
//! returning derivative events to be appended to the BEL.
|
//! returning derivative events to be appended to the BEL.
|
||||||
|
|
||||||
use crate::data_build_event::Event;
|
use crate::data_build_event::Event;
|
||||||
use crate::data_deps::{WantTimestamps, missing_deps_to_want_events};
|
use crate::data_deps::missing_deps_to_want_events;
|
||||||
use crate::event_source::Source as EventSourceVariant;
|
|
||||||
use crate::job_run_state::{JobRun, JobRunWithState, QueuedState as JobQueuedState};
|
use crate::job_run_state::{JobRun, JobRunWithState, QueuedState as JobQueuedState};
|
||||||
use crate::partition_state::{BuildingPartitionRef, Partition};
|
use crate::partition_state::{BuildingPartitionRef, Partition};
|
||||||
use crate::util::current_timestamp;
|
use crate::util::current_timestamp;
|
||||||
use crate::want_state::{NewState as WantNewState, Want, WantWithState};
|
use crate::want_create_event_v1::Lifetime;
|
||||||
|
use crate::want_state::{NewState as WantNewState, Want, WantLifetime, WantWithState};
|
||||||
use crate::{
|
use crate::{
|
||||||
JobRunBufferEventV1, JobRunCancelEventV1, JobRunFailureEventV1, JobRunHeartbeatEventV1,
|
JobRunBufferEventV1, JobRunCancelEventV1, JobRunFailureEventV1, JobRunHeartbeatEventV1,
|
||||||
JobRunMissingDepsEventV1, JobRunSuccessEventV1, PartitionRef, TaintCancelEventV1,
|
JobRunMissingDepsEventV1, JobRunSuccessEventV1, PartitionRef, TaintCancelEventV1,
|
||||||
|
|
@ -45,26 +45,22 @@ impl BuildState {
|
||||||
// Create want in New state from event
|
// Create want in New state from event
|
||||||
let want_new: WantWithState<WantNewState> = event.clone().into();
|
let want_new: WantWithState<WantNewState> = event.clone().into();
|
||||||
|
|
||||||
// Log creation with derivative vs user-created distinction
|
// Log creation with derivative vs user-created distinction based on lifetime
|
||||||
let is_derivative = if let Some(source) = &event.source {
|
let is_derivative = matches!(&event.lifetime, Some(Lifetime::Ephemeral(_)));
|
||||||
if let Some(EventSourceVariant::JobTriggered(job_triggered)) = &source.source {
|
|
||||||
tracing::info!(
|
if let Some(Lifetime::Ephemeral(eph)) = &event.lifetime {
|
||||||
want_id = %event.want_id,
|
tracing::info!(
|
||||||
partitions = ?event.partitions.iter().map(|p| &p.r#ref).collect::<Vec<_>>(),
|
want_id = %event.want_id,
|
||||||
source_job_run_id = %job_triggered.job_run_id,
|
partitions = ?event.partitions.iter().map(|p| &p.r#ref).collect::<Vec<_>>(),
|
||||||
"Want created (derivative - auto-created due to missing dependency)"
|
source_job_run_id = %eph.job_run_id,
|
||||||
);
|
"Want created (ephemeral - auto-created due to missing dependency)"
|
||||||
true
|
);
|
||||||
} else {
|
|
||||||
false
|
|
||||||
}
|
|
||||||
} else {
|
} else {
|
||||||
tracing::info!(
|
tracing::info!(
|
||||||
want_id = %event.want_id,
|
want_id = %event.want_id,
|
||||||
partitions = ?event.partitions.iter().map(|p| &p.r#ref).collect::<Vec<_>>(),
|
partitions = ?event.partitions.iter().map(|p| &p.r#ref).collect::<Vec<_>>(),
|
||||||
"Want created (user-requested)"
|
"Want created (originating - user-requested)"
|
||||||
);
|
);
|
||||||
false
|
|
||||||
};
|
};
|
||||||
|
|
||||||
// Register this want with all its partitions (via inverted index)
|
// Register this want with all its partitions (via inverted index)
|
||||||
|
|
@ -175,17 +171,22 @@ impl BuildState {
|
||||||
|
|
||||||
self.wants.insert(event.want_id.clone(), final_want);
|
self.wants.insert(event.want_id.clone(), final_want);
|
||||||
|
|
||||||
// If this is a derivative want (triggered by a job's dep miss), transition impacted wants to UpstreamBuilding
|
// If this is an ephemeral want (triggered by a job's dep miss):
|
||||||
if is_derivative {
|
// 1. Record the derivative want ID on the source job run
|
||||||
if let Some(source) = &event.source {
|
// 2. Transition impacted wants to UpstreamBuilding
|
||||||
if let Some(EventSourceVariant::JobTriggered(job_triggered)) = &source.source {
|
if let Some(Lifetime::Ephemeral(eph)) = &event.lifetime {
|
||||||
self.handle_derivative_want_creation(
|
// Add this want as a derivative of the source job run
|
||||||
&event.want_id,
|
if let Some(job_run) = self.job_runs.get_mut(&eph.job_run_id) {
|
||||||
&event.partitions,
|
if let JobRun::DepMiss(dep_miss) = job_run {
|
||||||
&job_triggered.job_run_id,
|
dep_miss.add_derivative_want_id(&event.want_id);
|
||||||
);
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
self.handle_derivative_want_creation(
|
||||||
|
&event.want_id,
|
||||||
|
&event.partitions,
|
||||||
|
&eph.job_run_id,
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
vec![]
|
vec![]
|
||||||
|
|
@ -242,7 +243,7 @@ impl BuildState {
|
||||||
// Create job run in Queued state
|
// Create job run in Queued state
|
||||||
let queued: JobRunWithState<JobQueuedState> = event.clone().into();
|
let queued: JobRunWithState<JobQueuedState> = event.clone().into();
|
||||||
|
|
||||||
// Transition wants to Building
|
// Transition wants to Building and track this job run on each want
|
||||||
// Valid states when job buffer event arrives:
|
// Valid states when job buffer event arrives:
|
||||||
// - Idle: First job starting for this want (normal case)
|
// - Idle: First job starting for this want (normal case)
|
||||||
// - Building: Another job already started for this want (multiple jobs can service same want)
|
// - Building: Another job already started for this want (multiple jobs can service same want)
|
||||||
|
|
@ -255,7 +256,7 @@ impl BuildState {
|
||||||
wap.want_id
|
wap.want_id
|
||||||
));
|
));
|
||||||
|
|
||||||
let transitioned = match want {
|
let mut transitioned = match want {
|
||||||
Want::New(new_want) => {
|
Want::New(new_want) => {
|
||||||
// Want was just created and hasn't fully sensed yet - transition to Building
|
// Want was just created and hasn't fully sensed yet - transition to Building
|
||||||
// This can happen if want creation and job buffer happen in quick succession
|
// This can happen if want creation and job buffer happen in quick succession
|
||||||
|
|
@ -287,6 +288,9 @@ impl BuildState {
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
|
// Track this job run on the want for lineage
|
||||||
|
transitioned.add_job_run_id(&event.job_run_id);
|
||||||
|
|
||||||
self.wants.insert(wap.want_id.clone(), transitioned);
|
self.wants.insert(wap.want_id.clone(), transitioned);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -340,18 +344,51 @@ impl BuildState {
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) fn handle_job_run_success(&mut self, event: &JobRunSuccessEventV1) -> Vec<Event> {
|
pub(crate) fn handle_job_run_success(&mut self, event: &JobRunSuccessEventV1) -> Vec<Event> {
|
||||||
|
use std::collections::BTreeMap;
|
||||||
|
|
||||||
let job_run = self.job_runs.remove(&event.job_run_id).expect(&format!(
|
let job_run = self.job_runs.remove(&event.job_run_id).expect(&format!(
|
||||||
"BUG: Job run {} must exist when success event received",
|
"BUG: Job run {} must exist when success event received",
|
||||||
event.job_run_id
|
event.job_run_id
|
||||||
));
|
));
|
||||||
|
|
||||||
|
// Resolve read partition UUIDs from the read_deps in the event
|
||||||
|
let mut read_partition_uuids: BTreeMap<String, String> = BTreeMap::new();
|
||||||
|
for read_dep in &event.read_deps {
|
||||||
|
for read_ref in &read_dep.read {
|
||||||
|
if let Some(uuid) = self.get_canonical_partition_uuid(&read_ref.r#ref) {
|
||||||
|
read_partition_uuids.insert(read_ref.r#ref.clone(), uuid.to_string());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Get the building partitions from the job run to resolve wrote_partition_uuids
|
||||||
|
let building_partitions = match &job_run {
|
||||||
|
JobRun::Running(running) => running.info.building_partitions.clone(),
|
||||||
|
_ => vec![],
|
||||||
|
};
|
||||||
|
|
||||||
|
// Resolve wrote partition UUIDs - these will be set when we transition partitions to Live
|
||||||
|
// For now, we compute them based on job_run_id (same logic as transition_partitions_to_live)
|
||||||
|
let mut wrote_partition_uuids: BTreeMap<String, String> = BTreeMap::new();
|
||||||
|
for pref in &building_partitions {
|
||||||
|
let uuid =
|
||||||
|
crate::partition_state::derive_partition_uuid(&event.job_run_id, &pref.r#ref);
|
||||||
|
wrote_partition_uuids.insert(pref.r#ref.clone(), uuid.to_string());
|
||||||
|
}
|
||||||
|
|
||||||
let succeeded = match job_run {
|
let succeeded = match job_run {
|
||||||
JobRun::Running(running) => {
|
JobRun::Running(running) => {
|
||||||
tracing::info!(
|
tracing::info!(
|
||||||
job_run_id = %event.job_run_id,
|
job_run_id = %event.job_run_id,
|
||||||
|
read_deps_count = event.read_deps.len(),
|
||||||
"JobRun: Running → Succeeded"
|
"JobRun: Running → Succeeded"
|
||||||
);
|
);
|
||||||
running.succeed(current_timestamp())
|
running.succeed(
|
||||||
|
current_timestamp(),
|
||||||
|
event.read_deps.clone(),
|
||||||
|
read_partition_uuids.clone(),
|
||||||
|
wrote_partition_uuids.clone(),
|
||||||
|
)
|
||||||
}
|
}
|
||||||
_ => {
|
_ => {
|
||||||
panic!(
|
panic!(
|
||||||
|
|
@ -361,6 +398,34 @@ impl BuildState {
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
|
// Populate the consumer index from read_deps (using UUIDs for historical lineage)
|
||||||
|
// For each read partition UUID, record that the written partition UUIDs consumed it
|
||||||
|
for read_dep in &event.read_deps {
|
||||||
|
for read_ref in &read_dep.read {
|
||||||
|
// Look up the read partition's UUID
|
||||||
|
if let Some(read_uuid) = read_partition_uuids.get(&read_ref.r#ref) {
|
||||||
|
if let Ok(read_uuid) = uuid::Uuid::parse_str(read_uuid) {
|
||||||
|
let consumers = self
|
||||||
|
.partition_consumers
|
||||||
|
.entry(read_uuid)
|
||||||
|
.or_insert_with(Vec::new);
|
||||||
|
for impacted_ref in &read_dep.impacted {
|
||||||
|
// Look up the impacted (output) partition's UUID
|
||||||
|
if let Some(wrote_uuid) = wrote_partition_uuids.get(&impacted_ref.r#ref)
|
||||||
|
{
|
||||||
|
if let Ok(wrote_uuid) = uuid::Uuid::parse_str(wrote_uuid) {
|
||||||
|
let entry = (wrote_uuid, event.job_run_id.clone());
|
||||||
|
if !consumers.contains(&entry) {
|
||||||
|
consumers.push(entry);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// Job run success is SOURCE of truth that partitions are live
|
// Job run success is SOURCE of truth that partitions are live
|
||||||
let newly_live_partitions = succeeded.get_completed_partitions();
|
let newly_live_partitions = succeeded.get_completed_partitions();
|
||||||
|
|
||||||
|
|
@ -514,15 +579,6 @@ impl BuildState {
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
// Infer data/SLA timestamps from servicing wants
|
|
||||||
let want_timestamps: WantTimestamps = dep_miss
|
|
||||||
.info
|
|
||||||
.servicing_wants
|
|
||||||
.iter()
|
|
||||||
.flat_map(|wap| self.get_want(&wap.want_id).map(|w| w.into()))
|
|
||||||
.reduce(|a: WantTimestamps, b: WantTimestamps| a.merge(b))
|
|
||||||
.expect("BUG: No servicing wants found");
|
|
||||||
|
|
||||||
// Collect all missing deps into a flat list of partition refs
|
// Collect all missing deps into a flat list of partition refs
|
||||||
let all_missing_deps: Vec<PartitionRef> = event
|
let all_missing_deps: Vec<PartitionRef> = event
|
||||||
.missing_deps
|
.missing_deps
|
||||||
|
|
@ -534,13 +590,11 @@ impl BuildState {
|
||||||
let building_refs_to_reset = dep_miss.get_building_partitions_to_reset();
|
let building_refs_to_reset = dep_miss.get_building_partitions_to_reset();
|
||||||
self.transition_partitions_to_upstream_building(&building_refs_to_reset, all_missing_deps);
|
self.transition_partitions_to_upstream_building(&building_refs_to_reset, all_missing_deps);
|
||||||
|
|
||||||
// Generate WantCreateV1 events for the missing dependencies
|
// Generate ephemeral WantCreateV1 events for the missing dependencies
|
||||||
// These events will be returned and appended to the BEL by BuildEventLog.append_event()
|
// These events will be returned and appended to the BEL by BuildEventLog.append_event()
|
||||||
let want_events = missing_deps_to_want_events(
|
// Ephemeral wants delegate freshness decisions to their originating want via the job_run_id reference
|
||||||
dep_miss.get_missing_deps().to_vec(),
|
let want_events =
|
||||||
&event.job_run_id,
|
missing_deps_to_want_events(dep_miss.get_missing_deps().to_vec(), &event.job_run_id);
|
||||||
want_timestamps,
|
|
||||||
);
|
|
||||||
|
|
||||||
// Store the job run in DepMiss state so we can access the missing_deps later
|
// Store the job run in DepMiss state so we can access the missing_deps later
|
||||||
// When the derivative WantCreateV1 events get processed by handle_want_create(),
|
// When the derivative WantCreateV1 events get processed by handle_want_create(),
|
||||||
|
|
@ -642,17 +696,22 @@ impl BuildState {
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
use super::*;
|
use super::*;
|
||||||
use crate::{MissingDeps, WantAttributedPartitions};
|
|
||||||
|
|
||||||
mod want {
|
mod want {
|
||||||
use super::*;
|
use super::*;
|
||||||
use crate::WantDetail;
|
use crate::want_create_event_v1::Lifetime;
|
||||||
|
use crate::{OriginatingLifetime, WantDetail};
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn test_should_create_want() {
|
fn test_should_create_want() {
|
||||||
let mut e = WantCreateEventV1::default();
|
let mut e = WantCreateEventV1::default();
|
||||||
e.want_id = "1234".to_string();
|
e.want_id = "1234".to_string();
|
||||||
e.partitions = vec!["mypart".into()];
|
e.partitions = vec!["mypart".into()];
|
||||||
|
e.lifetime = Some(Lifetime::Originating(OriginatingLifetime {
|
||||||
|
data_timestamp: 1000,
|
||||||
|
ttl_seconds: 3600,
|
||||||
|
sla_seconds: 7200,
|
||||||
|
}));
|
||||||
|
|
||||||
let mut state = BuildState::default();
|
let mut state = BuildState::default();
|
||||||
state.handle_event(&e.clone().into());
|
state.handle_event(&e.clone().into());
|
||||||
|
|
@ -665,9 +724,12 @@ mod tests {
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn test_should_cancel_want() {
|
fn test_should_cancel_want() {
|
||||||
|
use crate::util::test_scenarios::default_originating_lifetime;
|
||||||
|
|
||||||
let mut e = WantCreateEventV1::default();
|
let mut e = WantCreateEventV1::default();
|
||||||
e.want_id = "1234".to_string();
|
e.want_id = "1234".to_string();
|
||||||
e.partitions = vec!["mypart".into()];
|
e.partitions = vec!["mypart".into()];
|
||||||
|
e.lifetime = Some(default_originating_lifetime());
|
||||||
|
|
||||||
let mut state = BuildState::default();
|
let mut state = BuildState::default();
|
||||||
state.handle_event(&e.clone().into());
|
state.handle_event(&e.clone().into());
|
||||||
|
|
@ -686,133 +748,25 @@ mod tests {
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn test_multihop_dependency_replay() {
|
fn test_multihop_dependency_replay() {
|
||||||
use crate::{
|
use crate::util::test_scenarios::multihop_scenario;
|
||||||
JobRunBufferEventV1, JobRunHeartbeatEventV1, JobRunMissingDepsEventV1,
|
|
||||||
JobRunSuccessEventV1, MissingDeps, PartitionRef, WantAttributedPartitions,
|
|
||||||
WantCreateEventV1,
|
|
||||||
};
|
|
||||||
|
|
||||||
let mut state = BuildState::default();
|
let (events, ids) = multihop_scenario();
|
||||||
let mut events = vec![];
|
|
||||||
|
|
||||||
// 1. Create want for data/beta
|
|
||||||
let beta_want_id = "beta-want".to_string();
|
|
||||||
let mut create_beta = WantCreateEventV1::default();
|
|
||||||
create_beta.want_id = beta_want_id.clone();
|
|
||||||
create_beta.partitions = vec![PartitionRef {
|
|
||||||
r#ref: "data/beta".to_string(),
|
|
||||||
}];
|
|
||||||
events.push(Event::WantCreateV1(create_beta));
|
|
||||||
|
|
||||||
// 2. Queue beta job (first attempt)
|
|
||||||
let beta_job_1_id = "beta-job-1".to_string();
|
|
||||||
let mut buffer_beta_1 = JobRunBufferEventV1::default();
|
|
||||||
buffer_beta_1.job_run_id = beta_job_1_id.clone();
|
|
||||||
buffer_beta_1.job_label = "//job_beta".to_string();
|
|
||||||
buffer_beta_1.want_attributed_partitions = vec![WantAttributedPartitions {
|
|
||||||
want_id: beta_want_id.clone(),
|
|
||||||
partitions: vec![PartitionRef {
|
|
||||||
r#ref: "data/beta".to_string(),
|
|
||||||
}],
|
|
||||||
}];
|
|
||||||
buffer_beta_1.building_partitions = vec![PartitionRef {
|
|
||||||
r#ref: "data/beta".to_string(),
|
|
||||||
}];
|
|
||||||
events.push(Event::JobRunBufferV1(buffer_beta_1));
|
|
||||||
|
|
||||||
// 3. Beta job starts running
|
|
||||||
let mut heartbeat_beta_1 = JobRunHeartbeatEventV1::default();
|
|
||||||
heartbeat_beta_1.job_run_id = beta_job_1_id.clone();
|
|
||||||
events.push(Event::JobRunHeartbeatV1(heartbeat_beta_1));
|
|
||||||
|
|
||||||
// 4. Beta job reports missing dependency on data/alpha
|
|
||||||
let mut dep_miss_beta_1 = JobRunMissingDepsEventV1::default();
|
|
||||||
dep_miss_beta_1.job_run_id = beta_job_1_id.clone();
|
|
||||||
dep_miss_beta_1.missing_deps = vec![MissingDeps {
|
|
||||||
impacted: vec![PartitionRef {
|
|
||||||
r#ref: "data/beta".to_string(),
|
|
||||||
}],
|
|
||||||
missing: vec![PartitionRef {
|
|
||||||
r#ref: "data/alpha".to_string(),
|
|
||||||
}],
|
|
||||||
}];
|
|
||||||
events.push(Event::JobRunMissingDepsV1(dep_miss_beta_1));
|
|
||||||
|
|
||||||
// 5. Create derivative want for data/alpha
|
|
||||||
let alpha_want_id = "alpha-want".to_string();
|
|
||||||
let mut create_alpha = WantCreateEventV1::default();
|
|
||||||
create_alpha.want_id = alpha_want_id.clone();
|
|
||||||
create_alpha.partitions = vec![PartitionRef {
|
|
||||||
r#ref: "data/alpha".to_string(),
|
|
||||||
}];
|
|
||||||
events.push(Event::WantCreateV1(create_alpha));
|
|
||||||
|
|
||||||
// 6. Queue alpha job
|
|
||||||
let alpha_job_id = "alpha-job".to_string();
|
|
||||||
let mut buffer_alpha = JobRunBufferEventV1::default();
|
|
||||||
buffer_alpha.job_run_id = alpha_job_id.clone();
|
|
||||||
buffer_alpha.job_label = "//job_alpha".to_string();
|
|
||||||
buffer_alpha.want_attributed_partitions = vec![WantAttributedPartitions {
|
|
||||||
want_id: alpha_want_id.clone(),
|
|
||||||
partitions: vec![PartitionRef {
|
|
||||||
r#ref: "data/alpha".to_string(),
|
|
||||||
}],
|
|
||||||
}];
|
|
||||||
buffer_alpha.building_partitions = vec![PartitionRef {
|
|
||||||
r#ref: "data/alpha".to_string(),
|
|
||||||
}];
|
|
||||||
events.push(Event::JobRunBufferV1(buffer_alpha));
|
|
||||||
|
|
||||||
// 7. Alpha job starts running
|
|
||||||
let mut heartbeat_alpha = JobRunHeartbeatEventV1::default();
|
|
||||||
heartbeat_alpha.job_run_id = alpha_job_id.clone();
|
|
||||||
events.push(Event::JobRunHeartbeatV1(heartbeat_alpha));
|
|
||||||
|
|
||||||
// 8. Alpha job succeeds
|
|
||||||
let mut success_alpha = JobRunSuccessEventV1::default();
|
|
||||||
success_alpha.job_run_id = alpha_job_id.clone();
|
|
||||||
events.push(Event::JobRunSuccessV1(success_alpha));
|
|
||||||
|
|
||||||
// 9. Queue beta job again (second attempt) - THIS IS THE CRITICAL MOMENT
|
|
||||||
let beta_job_2_id = "beta-job-2".to_string();
|
|
||||||
let mut buffer_beta_2 = JobRunBufferEventV1::default();
|
|
||||||
buffer_beta_2.job_run_id = beta_job_2_id.clone();
|
|
||||||
buffer_beta_2.job_label = "//job_beta".to_string();
|
|
||||||
buffer_beta_2.want_attributed_partitions = vec![WantAttributedPartitions {
|
|
||||||
want_id: beta_want_id.clone(),
|
|
||||||
partitions: vec![PartitionRef {
|
|
||||||
r#ref: "data/beta".to_string(),
|
|
||||||
}],
|
|
||||||
}];
|
|
||||||
buffer_beta_2.building_partitions = vec![PartitionRef {
|
|
||||||
r#ref: "data/beta".to_string(),
|
|
||||||
}];
|
|
||||||
events.push(Event::JobRunBufferV1(buffer_beta_2));
|
|
||||||
|
|
||||||
// 10. Beta job starts running
|
|
||||||
let mut heartbeat_beta_2 = JobRunHeartbeatEventV1::default();
|
|
||||||
heartbeat_beta_2.job_run_id = beta_job_2_id.clone();
|
|
||||||
events.push(Event::JobRunHeartbeatV1(heartbeat_beta_2));
|
|
||||||
|
|
||||||
// 11. Beta job succeeds
|
|
||||||
let mut success_beta_2 = JobRunSuccessEventV1::default();
|
|
||||||
success_beta_2.job_run_id = beta_job_2_id.clone();
|
|
||||||
events.push(Event::JobRunSuccessV1(success_beta_2));
|
|
||||||
|
|
||||||
// Process all events - this simulates replay
|
// Process all events - this simulates replay
|
||||||
|
let mut state = BuildState::default();
|
||||||
for event in &events {
|
for event in &events {
|
||||||
state.handle_event(event);
|
state.handle_event(event);
|
||||||
}
|
}
|
||||||
|
|
||||||
// Verify final state
|
// Verify final state
|
||||||
let beta_want = state.get_want(&beta_want_id).unwrap();
|
let beta_want = state.get_want(&ids.beta_want_id).unwrap();
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
beta_want.status,
|
beta_want.status,
|
||||||
Some(crate::WantStatusCode::WantSuccessful.into()),
|
Some(crate::WantStatusCode::WantSuccessful.into()),
|
||||||
"Beta want should be successful after multi-hop dependency resolution"
|
"Beta want should be successful after multi-hop dependency resolution"
|
||||||
);
|
);
|
||||||
|
|
||||||
let alpha_want = state.get_want(&alpha_want_id).unwrap();
|
let alpha_want = state.get_want(&ids.alpha_want_id).unwrap();
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
alpha_want.status,
|
alpha_want.status,
|
||||||
Some(crate::WantStatusCode::WantSuccessful.into()),
|
Some(crate::WantStatusCode::WantSuccessful.into()),
|
||||||
|
|
@ -824,6 +778,7 @@ mod tests {
|
||||||
/// This was the original bug that motivated the UUID refactor.
|
/// This was the original bug that motivated the UUID refactor.
|
||||||
#[test]
|
#[test]
|
||||||
fn test_concurrent_wants_same_partition() {
|
fn test_concurrent_wants_same_partition() {
|
||||||
|
use crate::util::test_scenarios::default_originating_lifetime;
|
||||||
use crate::{
|
use crate::{
|
||||||
JobRunBufferEventV1, JobRunHeartbeatEventV1, PartitionRef,
|
JobRunBufferEventV1, JobRunHeartbeatEventV1, PartitionRef,
|
||||||
WantAttributedPartitions, WantCreateEventV1,
|
WantAttributedPartitions, WantCreateEventV1,
|
||||||
|
|
@ -838,6 +793,7 @@ mod tests {
|
||||||
create_want_1.partitions = vec![PartitionRef {
|
create_want_1.partitions = vec![PartitionRef {
|
||||||
r#ref: "data/beta".to_string(),
|
r#ref: "data/beta".to_string(),
|
||||||
}];
|
}];
|
||||||
|
create_want_1.lifetime = Some(default_originating_lifetime());
|
||||||
state.handle_event(&Event::WantCreateV1(create_want_1));
|
state.handle_event(&Event::WantCreateV1(create_want_1));
|
||||||
|
|
||||||
// Want 1 should be Idle (no partition exists yet)
|
// Want 1 should be Idle (no partition exists yet)
|
||||||
|
|
@ -855,6 +811,7 @@ mod tests {
|
||||||
create_want_2.partitions = vec![PartitionRef {
|
create_want_2.partitions = vec![PartitionRef {
|
||||||
r#ref: "data/beta".to_string(),
|
r#ref: "data/beta".to_string(),
|
||||||
}];
|
}];
|
||||||
|
create_want_2.lifetime = Some(default_originating_lifetime());
|
||||||
state.handle_event(&Event::WantCreateV1(create_want_2));
|
state.handle_event(&Event::WantCreateV1(create_want_2));
|
||||||
|
|
||||||
// Want 2 should also be Idle
|
// Want 2 should also be Idle
|
||||||
|
|
@ -932,6 +889,7 @@ mod tests {
|
||||||
|
|
||||||
mod partition_lifecycle {
|
mod partition_lifecycle {
|
||||||
use super::*;
|
use super::*;
|
||||||
|
use crate::util::test_scenarios::default_originating_lifetime;
|
||||||
use crate::{
|
use crate::{
|
||||||
JobRunBufferEventV1, JobRunFailureEventV1, JobRunHeartbeatEventV1,
|
JobRunBufferEventV1, JobRunFailureEventV1, JobRunHeartbeatEventV1,
|
||||||
JobRunMissingDepsEventV1, JobRunSuccessEventV1, MissingDeps, PartitionRef,
|
JobRunMissingDepsEventV1, JobRunSuccessEventV1, MissingDeps, PartitionRef,
|
||||||
|
|
@ -952,6 +910,7 @@ mod tests {
|
||||||
create_beta.partitions = vec![PartitionRef {
|
create_beta.partitions = vec![PartitionRef {
|
||||||
r#ref: "data/beta".to_string(),
|
r#ref: "data/beta".to_string(),
|
||||||
}];
|
}];
|
||||||
|
create_beta.lifetime = Some(default_originating_lifetime());
|
||||||
state.handle_event(&Event::WantCreateV1(create_beta));
|
state.handle_event(&Event::WantCreateV1(create_beta));
|
||||||
|
|
||||||
// 2. Job buffers for beta
|
// 2. Job buffers for beta
|
||||||
|
|
@ -1082,6 +1041,7 @@ mod tests {
|
||||||
create_beta.partitions = vec![PartitionRef {
|
create_beta.partitions = vec![PartitionRef {
|
||||||
r#ref: "data/beta".to_string(),
|
r#ref: "data/beta".to_string(),
|
||||||
}];
|
}];
|
||||||
|
create_beta.lifetime = Some(default_originating_lifetime());
|
||||||
state.handle_event(&Event::WantCreateV1(create_beta));
|
state.handle_event(&Event::WantCreateV1(create_beta));
|
||||||
|
|
||||||
// 2. First job buffers for beta (creates uuid-1)
|
// 2. First job buffers for beta (creates uuid-1)
|
||||||
|
|
@ -1129,6 +1089,7 @@ mod tests {
|
||||||
create_alpha.partitions = vec![PartitionRef {
|
create_alpha.partitions = vec![PartitionRef {
|
||||||
r#ref: "data/alpha".to_string(),
|
r#ref: "data/alpha".to_string(),
|
||||||
}];
|
}];
|
||||||
|
create_alpha.lifetime = Some(default_originating_lifetime());
|
||||||
state.handle_event(&Event::WantCreateV1(create_alpha));
|
state.handle_event(&Event::WantCreateV1(create_alpha));
|
||||||
|
|
||||||
let alpha_job_id = "alpha-job".to_string();
|
let alpha_job_id = "alpha-job".to_string();
|
||||||
|
|
|
||||||
|
|
@ -61,6 +61,11 @@ pub struct BuildState {
|
||||||
// Inverted indexes
|
// Inverted indexes
|
||||||
pub(crate) wants_for_partition: BTreeMap<String, Vec<String>>, // partition ref → want_ids
|
pub(crate) wants_for_partition: BTreeMap<String, Vec<String>>, // partition ref → want_ids
|
||||||
pub(crate) downstream_waiting: BTreeMap<String, Vec<Uuid>>, // upstream ref → partition UUIDs waiting for it
|
pub(crate) downstream_waiting: BTreeMap<String, Vec<Uuid>>, // upstream ref → partition UUIDs waiting for it
|
||||||
|
|
||||||
|
// Consumer index for lineage queries: input_uuid → list of (output_uuid, job_run_id)
|
||||||
|
// Uses UUIDs (not refs) to preserve historical lineage across partition rebuilds
|
||||||
|
// Populated from read_deps on job success
|
||||||
|
pub(crate) partition_consumers: BTreeMap<Uuid, Vec<(Uuid, String)>>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl BuildState {
|
impl BuildState {
|
||||||
|
|
@ -118,6 +123,15 @@ impl BuildState {
|
||||||
.unwrap_or(&[])
|
.unwrap_or(&[])
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Get consumers for a partition UUID (downstream partitions that read this one)
|
||||||
|
/// Returns list of (output_uuid, job_run_id) tuples
|
||||||
|
pub fn get_partition_consumers(&self, uuid: &Uuid) -> &[(Uuid, String)] {
|
||||||
|
self.partition_consumers
|
||||||
|
.get(uuid)
|
||||||
|
.map(|v| v.as_slice())
|
||||||
|
.unwrap_or(&[])
|
||||||
|
}
|
||||||
|
|
||||||
/// Register a want in the wants_for_partition inverted index
|
/// Register a want in the wants_for_partition inverted index
|
||||||
pub(crate) fn register_want_for_partitions(
|
pub(crate) fn register_want_for_partitions(
|
||||||
&mut self,
|
&mut self,
|
||||||
|
|
|
||||||
|
|
@ -2,10 +2,12 @@
|
||||||
//!
|
//!
|
||||||
//! Read-only methods for accessing state (get_*, list_*) used by the API layer.
|
//! Read-only methods for accessing state (get_*, list_*) used by the API layer.
|
||||||
|
|
||||||
|
use crate::util::{HasRelatedIds, RelatedIds};
|
||||||
use crate::{
|
use crate::{
|
||||||
JobRunDetail, ListJobRunsRequest, ListJobRunsResponse, ListPartitionsRequest,
|
GetJobRunResponse, GetPartitionResponse, GetWantResponse, JobRunDetail, ListJobRunsRequest,
|
||||||
ListPartitionsResponse, ListTaintsRequest, ListTaintsResponse, ListWantsRequest,
|
ListJobRunsResponse, ListPartitionsRequest, ListPartitionsResponse, ListTaintsRequest,
|
||||||
ListWantsResponse, PartitionDetail, TaintDetail, WantDetail,
|
ListTaintsResponse, ListWantsRequest, ListWantsResponse, PartitionDetail, RelatedEntities,
|
||||||
|
TaintDetail, WantDetail,
|
||||||
};
|
};
|
||||||
use std::collections::BTreeMap;
|
use std::collections::BTreeMap;
|
||||||
|
|
||||||
|
|
@ -13,7 +15,34 @@ use super::{BuildState, consts};
|
||||||
|
|
||||||
impl BuildState {
|
impl BuildState {
|
||||||
pub fn get_want(&self, want_id: &str) -> Option<WantDetail> {
|
pub fn get_want(&self, want_id: &str) -> Option<WantDetail> {
|
||||||
self.wants.get(want_id).map(|w| w.to_detail())
|
self.wants.get(want_id).map(|w| {
|
||||||
|
let mut detail = w.to_detail();
|
||||||
|
// Populate job_runs and compute derivative_want_ids by traversing job runs.
|
||||||
|
//
|
||||||
|
// derivative_want_ids is computed at query time rather than maintained during
|
||||||
|
// event handling. The relationship flows: Want → JobRun → (dep-miss) → EphemeralWant
|
||||||
|
//
|
||||||
|
// - JobRun tracks which derivative wants it spawned (on DepMissState)
|
||||||
|
// - Want only tracks which job runs serviced it (job_run_ids)
|
||||||
|
// - At query time, we traverse: Want's job_run_ids → each JobRun's derivative_want_ids
|
||||||
|
//
|
||||||
|
// This keeps event handling simple (just update the job run) and keeps JobRun
|
||||||
|
// as the source of truth for derivative want relationships.
|
||||||
|
for job_run_id in &detail.job_run_ids {
|
||||||
|
if let Some(job_run) = self.job_runs.get(job_run_id) {
|
||||||
|
let job_detail = job_run.to_detail();
|
||||||
|
// Collect derivative want IDs
|
||||||
|
for derivative_want_id in &job_detail.derivative_want_ids {
|
||||||
|
if !detail.derivative_want_ids.contains(derivative_want_id) {
|
||||||
|
detail.derivative_want_ids.push(derivative_want_id.clone());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// Add full job run details
|
||||||
|
detail.job_runs.push(job_detail);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
detail
|
||||||
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn get_taint(&self, taint_id: &str) -> Option<TaintDetail> {
|
pub fn get_taint(&self, taint_id: &str) -> Option<TaintDetail> {
|
||||||
|
|
@ -49,6 +78,7 @@ impl BuildState {
|
||||||
match_count: self.wants.len() as u64,
|
match_count: self.wants.len() as u64,
|
||||||
page,
|
page,
|
||||||
page_size,
|
page_size,
|
||||||
|
index: None,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -60,6 +90,7 @@ impl BuildState {
|
||||||
match_count: self.wants.len() as u64,
|
match_count: self.wants.len() as u64,
|
||||||
page,
|
page,
|
||||||
page_size,
|
page_size,
|
||||||
|
index: None,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -81,6 +112,7 @@ impl BuildState {
|
||||||
match_count: self.canonical_partitions.len() as u64,
|
match_count: self.canonical_partitions.len() as u64,
|
||||||
page,
|
page,
|
||||||
page_size,
|
page_size,
|
||||||
|
index: None,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -102,6 +134,7 @@ impl BuildState {
|
||||||
match_count: self.job_runs.len() as u64,
|
match_count: self.job_runs.len() as u64,
|
||||||
page,
|
page,
|
||||||
page_size,
|
page_size,
|
||||||
|
index: None,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -116,3 +149,214 @@ fn list_state_items<T: Clone>(map: &BTreeMap<String, T>, page: u64, page_size: u
|
||||||
.cloned()
|
.cloned()
|
||||||
.collect()
|
.collect()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ============================================================================
|
||||||
|
// Response builders with RelatedEntities index
|
||||||
|
// ============================================================================
|
||||||
|
|
||||||
|
impl BuildState {
|
||||||
|
/// Resolve RelatedIds to a RelatedEntities index by looking up entities in BuildState.
|
||||||
|
/// This is the central method for building the index from collected IDs.
|
||||||
|
pub fn resolve_related_ids(&self, ids: &RelatedIds) -> RelatedEntities {
|
||||||
|
let mut index = RelatedEntities::default();
|
||||||
|
|
||||||
|
// Resolve partition refs
|
||||||
|
for partition_ref in &ids.partition_refs {
|
||||||
|
if !index.partitions.contains_key(partition_ref) {
|
||||||
|
if let Some(p) = self.get_canonical_partition(partition_ref) {
|
||||||
|
index
|
||||||
|
.partitions
|
||||||
|
.insert(partition_ref.clone(), p.to_detail());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Resolve partition UUIDs
|
||||||
|
for uuid in &ids.partition_uuids {
|
||||||
|
if let Some(p) = self.partitions_by_uuid.get(uuid) {
|
||||||
|
let detail = p.to_detail();
|
||||||
|
if let Some(ref pref) = detail.r#ref {
|
||||||
|
if !index.partitions.contains_key(&pref.r#ref) {
|
||||||
|
index.partitions.insert(pref.r#ref.clone(), detail);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Resolve job run IDs
|
||||||
|
for job_run_id in &ids.job_run_ids {
|
||||||
|
if !index.job_runs.contains_key(job_run_id) {
|
||||||
|
if let Some(jr) = self.job_runs.get(job_run_id) {
|
||||||
|
index.job_runs.insert(job_run_id.clone(), jr.to_detail());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Resolve want IDs
|
||||||
|
for want_id in &ids.want_ids {
|
||||||
|
if !index.wants.contains_key(want_id) {
|
||||||
|
if let Some(w) = self.wants.get(want_id) {
|
||||||
|
index.wants.insert(want_id.clone(), w.to_detail());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
index
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Get a want with its related entities (job runs, partitions)
|
||||||
|
pub fn get_want_with_index(&self, want_id: &str) -> Option<GetWantResponse> {
|
||||||
|
let want = self.wants.get(want_id)?;
|
||||||
|
let want_detail = want.to_detail();
|
||||||
|
let ids = want.related_ids();
|
||||||
|
let index = self.resolve_related_ids(&ids);
|
||||||
|
|
||||||
|
Some(GetWantResponse {
|
||||||
|
data: Some(want_detail),
|
||||||
|
index: Some(index),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Get a partition with its related entities (builder job run, downstream consumers)
|
||||||
|
pub fn get_partition_with_index(&self, partition_ref: &str) -> Option<GetPartitionResponse> {
|
||||||
|
let partition = self.get_canonical_partition(partition_ref)?;
|
||||||
|
let partition_detail = partition.to_detail();
|
||||||
|
|
||||||
|
let mut ids = partition.related_ids();
|
||||||
|
|
||||||
|
// Add downstream consumers from the consumer index (not stored on partition)
|
||||||
|
let uuid = partition.uuid();
|
||||||
|
for (output_uuid, job_run_id) in self.get_partition_consumers(&uuid) {
|
||||||
|
if !ids.partition_uuids.contains(output_uuid) {
|
||||||
|
ids.partition_uuids.push(*output_uuid);
|
||||||
|
}
|
||||||
|
if !ids.job_run_ids.contains(job_run_id) {
|
||||||
|
ids.job_run_ids.push(job_run_id.clone());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Add wants that reference this partition (from inverted index)
|
||||||
|
for want_id in self.get_wants_for_partition(partition_ref) {
|
||||||
|
if !ids.want_ids.contains(want_id) {
|
||||||
|
ids.want_ids.push(want_id.clone());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let index = self.resolve_related_ids(&ids);
|
||||||
|
|
||||||
|
Some(GetPartitionResponse {
|
||||||
|
data: Some(partition_detail),
|
||||||
|
index: Some(index),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Get a job run with its related entities (read/wrote partitions, derivative wants)
|
||||||
|
pub fn get_job_run_with_index(&self, job_run_id: &str) -> Option<GetJobRunResponse> {
|
||||||
|
let job_run = self.job_runs.get(job_run_id)?;
|
||||||
|
let job_run_detail = job_run.to_detail();
|
||||||
|
let ids = job_run.related_ids();
|
||||||
|
let index = self.resolve_related_ids(&ids);
|
||||||
|
|
||||||
|
Some(GetJobRunResponse {
|
||||||
|
data: Some(job_run_detail),
|
||||||
|
index: Some(index),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
/// List wants with related entities index
|
||||||
|
pub fn list_wants_with_index(&self, request: &ListWantsRequest) -> ListWantsResponse {
|
||||||
|
let page = request.page.unwrap_or(0);
|
||||||
|
let page_size = request.page_size.unwrap_or(consts::DEFAULT_PAGE_SIZE);
|
||||||
|
let start = page * page_size;
|
||||||
|
|
||||||
|
let wants: Vec<_> = self
|
||||||
|
.wants
|
||||||
|
.values()
|
||||||
|
.skip(start as usize)
|
||||||
|
.take(page_size as usize)
|
||||||
|
.collect();
|
||||||
|
|
||||||
|
// Collect related IDs from all wants
|
||||||
|
let mut all_ids = RelatedIds::default();
|
||||||
|
for want in &wants {
|
||||||
|
all_ids.merge(want.related_ids());
|
||||||
|
}
|
||||||
|
|
||||||
|
let data: Vec<WantDetail> = wants.iter().map(|w| w.to_detail()).collect();
|
||||||
|
let index = self.resolve_related_ids(&all_ids);
|
||||||
|
|
||||||
|
ListWantsResponse {
|
||||||
|
data,
|
||||||
|
match_count: self.wants.len() as u64,
|
||||||
|
page,
|
||||||
|
page_size,
|
||||||
|
index: Some(index),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// List partitions with related entities index
|
||||||
|
pub fn list_partitions_with_index(
|
||||||
|
&self,
|
||||||
|
request: &ListPartitionsRequest,
|
||||||
|
) -> ListPartitionsResponse {
|
||||||
|
let page = request.page.unwrap_or(0);
|
||||||
|
let page_size = request.page_size.unwrap_or(consts::DEFAULT_PAGE_SIZE);
|
||||||
|
let start = page * page_size;
|
||||||
|
|
||||||
|
let partitions: Vec<_> = self
|
||||||
|
.canonical_partitions
|
||||||
|
.iter()
|
||||||
|
.skip(start as usize)
|
||||||
|
.take(page_size as usize)
|
||||||
|
.filter_map(|(_, uuid)| self.partitions_by_uuid.get(uuid))
|
||||||
|
.collect();
|
||||||
|
|
||||||
|
// Collect related IDs from all partitions
|
||||||
|
let mut all_ids = RelatedIds::default();
|
||||||
|
for partition in &partitions {
|
||||||
|
all_ids.merge(partition.related_ids());
|
||||||
|
}
|
||||||
|
|
||||||
|
let data: Vec<PartitionDetail> = partitions.iter().map(|p| p.to_detail()).collect();
|
||||||
|
let index = self.resolve_related_ids(&all_ids);
|
||||||
|
|
||||||
|
ListPartitionsResponse {
|
||||||
|
data,
|
||||||
|
match_count: self.canonical_partitions.len() as u64,
|
||||||
|
page,
|
||||||
|
page_size,
|
||||||
|
index: Some(index),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// List job runs with related entities index
|
||||||
|
pub fn list_job_runs_with_index(&self, request: &ListJobRunsRequest) -> ListJobRunsResponse {
|
||||||
|
let page = request.page.unwrap_or(0);
|
||||||
|
let page_size = request.page_size.unwrap_or(consts::DEFAULT_PAGE_SIZE);
|
||||||
|
let start = page * page_size;
|
||||||
|
|
||||||
|
let job_runs: Vec<_> = self
|
||||||
|
.job_runs
|
||||||
|
.values()
|
||||||
|
.skip(start as usize)
|
||||||
|
.take(page_size as usize)
|
||||||
|
.collect();
|
||||||
|
|
||||||
|
// Collect related IDs from all job runs
|
||||||
|
let mut all_ids = RelatedIds::default();
|
||||||
|
for job_run in &job_runs {
|
||||||
|
all_ids.merge(job_run.related_ids());
|
||||||
|
}
|
||||||
|
|
||||||
|
let data: Vec<JobRunDetail> = job_runs.iter().map(|jr| jr.to_detail()).collect();
|
||||||
|
let index = self.resolve_related_ids(&all_ids);
|
||||||
|
|
||||||
|
ListJobRunsResponse {
|
||||||
|
data,
|
||||||
|
match_count: self.job_runs.len() as u64,
|
||||||
|
page,
|
||||||
|
page_size,
|
||||||
|
index: Some(index),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
|
||||||
|
|
@ -1,7 +1,7 @@
|
||||||
use crate::data_build_event::Event;
|
use crate::data_build_event::Event;
|
||||||
|
use crate::want_create_event_v1::Lifetime;
|
||||||
use crate::{
|
use crate::{
|
||||||
JobRunMissingDeps, JobRunReadDeps, JobTriggeredEvent, MissingDeps, ReadDeps, WantCreateEventV1,
|
EphemeralLifetime, JobRunMissingDeps, JobRunReadDeps, MissingDeps, ReadDeps, WantCreateEventV1,
|
||||||
WantDetail,
|
|
||||||
};
|
};
|
||||||
use uuid::Uuid;
|
use uuid::Uuid;
|
||||||
|
|
||||||
|
|
@ -82,53 +82,19 @@ fn line_matches<'a>(line: &'a str, prefix: &'a str) -> Option<&'a str> {
|
||||||
line.trim().strip_prefix(prefix)
|
line.trim().strip_prefix(prefix)
|
||||||
}
|
}
|
||||||
|
|
||||||
pub struct WantTimestamps {
|
/// Create ephemeral want events from missing dependencies.
|
||||||
data_timestamp: u64,
|
/// Ephemeral wants are derivative wants created by the system when a job hits a dep-miss.
|
||||||
ttl_seconds: u64,
|
/// They delegate freshness decisions to their originating want.
|
||||||
sla_seconds: u64,
|
pub fn missing_deps_to_want_events(missing_deps: Vec<MissingDeps>, job_run_id: &str) -> Vec<Event> {
|
||||||
}
|
|
||||||
|
|
||||||
impl From<WantDetail> for WantTimestamps {
|
|
||||||
fn from(want_detail: WantDetail) -> Self {
|
|
||||||
WantTimestamps {
|
|
||||||
data_timestamp: want_detail.data_timestamp,
|
|
||||||
ttl_seconds: want_detail.ttl_seconds,
|
|
||||||
sla_seconds: want_detail.sla_seconds,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
impl WantTimestamps {
|
|
||||||
pub fn merge(self, other: WantTimestamps) -> WantTimestamps {
|
|
||||||
// TODO does this make sense?
|
|
||||||
WantTimestamps {
|
|
||||||
data_timestamp: self.data_timestamp.min(other.data_timestamp),
|
|
||||||
ttl_seconds: self.ttl_seconds.max(other.ttl_seconds),
|
|
||||||
sla_seconds: self.sla_seconds.max(other.sla_seconds),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn missing_deps_to_want_events(
|
|
||||||
missing_deps: Vec<MissingDeps>,
|
|
||||||
job_run_id: &String,
|
|
||||||
want_timestamps: WantTimestamps,
|
|
||||||
) -> Vec<Event> {
|
|
||||||
missing_deps
|
missing_deps
|
||||||
.iter()
|
.iter()
|
||||||
.map(|md| {
|
.map(|md| {
|
||||||
Event::WantCreateV1(WantCreateEventV1 {
|
Event::WantCreateV1(WantCreateEventV1 {
|
||||||
want_id: Uuid::new_v4().into(),
|
want_id: Uuid::new_v4().into(),
|
||||||
partitions: md.missing.clone(),
|
partitions: md.missing.clone(),
|
||||||
data_timestamp: want_timestamps.data_timestamp,
|
lifetime: Some(Lifetime::Ephemeral(EphemeralLifetime {
|
||||||
ttl_seconds: want_timestamps.ttl_seconds,
|
job_run_id: job_run_id.to_string(),
|
||||||
sla_seconds: want_timestamps.sla_seconds,
|
})),
|
||||||
source: Some(
|
|
||||||
JobTriggeredEvent {
|
|
||||||
job_run_id: job_run_id.clone(),
|
|
||||||
}
|
|
||||||
.into(),
|
|
||||||
),
|
|
||||||
comment: Some("Missing data".to_string()),
|
comment: Some("Missing data".to_string()),
|
||||||
})
|
})
|
||||||
})
|
})
|
||||||
|
|
|
||||||
|
|
@ -62,6 +62,20 @@ message JobTriggeredEvent {
|
||||||
string job_run_id = 1;
|
string job_run_id = 1;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Want lifetime semantics
|
||||||
|
// Originating wants are user-created with explicit freshness requirements
|
||||||
|
message OriginatingLifetime {
|
||||||
|
uint64 data_timestamp = 1;
|
||||||
|
uint64 ttl_seconds = 2;
|
||||||
|
uint64 sla_seconds = 3;
|
||||||
|
}
|
||||||
|
// Ephemeral wants are system-created (derivative) from dep-miss
|
||||||
|
// They delegate freshness decisions to their originating want
|
||||||
|
message EphemeralLifetime {
|
||||||
|
// The job run that hit dep-miss and created this derivative want
|
||||||
|
string job_run_id = 1;
|
||||||
|
}
|
||||||
|
|
||||||
message WantAttributedPartitions {
|
message WantAttributedPartitions {
|
||||||
string want_id = 1;
|
string want_id = 1;
|
||||||
repeated PartitionRef partitions = 2;
|
repeated PartitionRef partitions = 2;
|
||||||
|
|
@ -85,9 +99,11 @@ message JobRunHeartbeatEventV1 {
|
||||||
string job_run_id = 1;
|
string job_run_id = 1;
|
||||||
// TODO reentrance?
|
// TODO reentrance?
|
||||||
}
|
}
|
||||||
// Simply indicates that the job has succeeded.
|
// Indicates that the job has succeeded, including what data was read.
|
||||||
message JobRunSuccessEventV1 {
|
message JobRunSuccessEventV1 {
|
||||||
string job_run_id = 1;
|
string job_run_id = 1;
|
||||||
|
// The read dependencies for this job run, preserving impacted→read relationships
|
||||||
|
repeated ReadDeps read_deps = 2;
|
||||||
}
|
}
|
||||||
// Simply indicates that the job has failed. Depending on retry logic defined in job, it may retry.
|
// Simply indicates that the job has failed. Depending on retry logic defined in job, it may retry.
|
||||||
message JobRunFailureEventV1 {
|
message JobRunFailureEventV1 {
|
||||||
|
|
@ -134,12 +150,14 @@ message WantCreateEventV1 {
|
||||||
// The unique ID of this want
|
// The unique ID of this want
|
||||||
string want_id = 1;
|
string want_id = 1;
|
||||||
repeated PartitionRef partitions = 2;
|
repeated PartitionRef partitions = 2;
|
||||||
uint64 data_timestamp = 3;
|
|
||||||
uint64 ttl_seconds = 4;
|
// Lifetime semantics - exactly one must be set
|
||||||
uint64 sla_seconds = 5;
|
oneof lifetime {
|
||||||
// The source of the want. Can be from job, API, CLI, web app...
|
OriginatingLifetime originating = 3;
|
||||||
EventSource source = 6;
|
EphemeralLifetime ephemeral = 4;
|
||||||
optional string comment = 7;
|
}
|
||||||
|
|
||||||
|
optional string comment = 5;
|
||||||
}
|
}
|
||||||
message WantCancelEventV1 {
|
message WantCancelEventV1 {
|
||||||
string want_id = 1;
|
string want_id = 1;
|
||||||
|
|
@ -205,14 +223,22 @@ message WantDetail {
|
||||||
repeated PartitionRef partitions = 2;
|
repeated PartitionRef partitions = 2;
|
||||||
// The upstream partitions, detected from a dep miss job run failure
|
// The upstream partitions, detected from a dep miss job run failure
|
||||||
repeated PartitionRef upstreams = 3;
|
repeated PartitionRef upstreams = 3;
|
||||||
uint64 data_timestamp = 4;
|
|
||||||
uint64 ttl_seconds = 5;
|
// Lifetime semantics
|
||||||
uint64 sla_seconds = 6;
|
oneof lifetime {
|
||||||
EventSource source = 7;
|
OriginatingLifetime originating = 4;
|
||||||
optional string comment = 8;
|
EphemeralLifetime ephemeral = 5;
|
||||||
WantStatus status = 9;
|
}
|
||||||
uint64 last_updated_timestamp = 10;
|
|
||||||
// TODO
|
optional string comment = 6;
|
||||||
|
WantStatus status = 7;
|
||||||
|
uint64 last_updated_timestamp = 8;
|
||||||
|
// Lineage: all job runs that have serviced this want (IDs for reference)
|
||||||
|
repeated string job_run_ids = 9;
|
||||||
|
// Lineage: derivative wants spawned by this want's job dep-misses (computed from job_run_ids)
|
||||||
|
repeated string derivative_want_ids = 10;
|
||||||
|
// Lineage: full details of job runs servicing this want (for display in tables)
|
||||||
|
repeated JobRunDetail job_runs = 11;
|
||||||
}
|
}
|
||||||
|
|
||||||
message PartitionDetail {
|
message PartitionDetail {
|
||||||
|
|
@ -230,6 +256,11 @@ message PartitionDetail {
|
||||||
// The unique identifier for this partition instance (UUID as string)
|
// The unique identifier for this partition instance (UUID as string)
|
||||||
// Each time a partition is built, it gets a new UUID derived from the job_run_id
|
// Each time a partition is built, it gets a new UUID derived from the job_run_id
|
||||||
string uuid = 7;
|
string uuid = 7;
|
||||||
|
// Lineage: job run that built this partition (for Live/Tainted partitions)
|
||||||
|
// Upstream lineage is resolved via this job run's read_deps (job run is source of truth)
|
||||||
|
optional string built_by_job_run_id = 8;
|
||||||
|
// Lineage: downstream partition UUIDs that consumed this (from consumer index)
|
||||||
|
repeated string downstream_partition_uuids = 9;
|
||||||
}
|
}
|
||||||
message PartitionStatus {
|
message PartitionStatus {
|
||||||
PartitionStatusCode code = 1;
|
PartitionStatusCode code = 1;
|
||||||
|
|
@ -289,10 +320,31 @@ message JobRunDetail {
|
||||||
optional uint64 last_heartbeat_at = 3;
|
optional uint64 last_heartbeat_at = 3;
|
||||||
repeated PartitionRef building_partitions = 4;
|
repeated PartitionRef building_partitions = 4;
|
||||||
repeated WantAttributedPartitions servicing_wants = 5;
|
repeated WantAttributedPartitions servicing_wants = 5;
|
||||||
|
// Lineage: read dependencies with resolved UUIDs (for Succeeded jobs)
|
||||||
|
repeated ReadDeps read_deps = 6;
|
||||||
|
// Lineage: resolved UUIDs for read partitions (ref → UUID)
|
||||||
|
map<string, string> read_partition_uuids = 7;
|
||||||
|
// Lineage: resolved UUIDs for written partitions (ref → UUID)
|
||||||
|
map<string, string> wrote_partition_uuids = 8;
|
||||||
|
// Lineage: derivative wants spawned by this job's dep-miss (for DepMiss jobs)
|
||||||
|
repeated string derivative_want_ids = 9;
|
||||||
|
// Timestamps for tracking job lifecycle
|
||||||
|
optional uint64 queued_at = 10;
|
||||||
|
optional uint64 started_at = 11;
|
||||||
|
// The job label (e.g. "//path/to:job")
|
||||||
|
string job_label = 12;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
// Related entities index - used in API responses for deduplication and O(1) lookup
|
||||||
|
// Each entity appears once in the index, even if referenced by multiple items in `data`
|
||||||
|
message RelatedEntities {
|
||||||
|
map<string, PartitionDetail> partitions = 1;
|
||||||
|
map<string, JobRunDetail> job_runs = 2;
|
||||||
|
map<string, WantDetail> wants = 3;
|
||||||
|
}
|
||||||
|
|
||||||
message EventFilter {
|
message EventFilter {
|
||||||
// IDs of wants to get relevant events for
|
// IDs of wants to get relevant events for
|
||||||
repeated string want_ids = 1;
|
repeated string want_ids = 1;
|
||||||
|
|
@ -308,6 +360,7 @@ message ListWantsResponse {
|
||||||
uint64 match_count = 2;
|
uint64 match_count = 2;
|
||||||
uint64 page = 3;
|
uint64 page = 3;
|
||||||
uint64 page_size = 4;
|
uint64 page_size = 4;
|
||||||
|
RelatedEntities index = 5;
|
||||||
}
|
}
|
||||||
|
|
||||||
message ListTaintsRequest {
|
message ListTaintsRequest {
|
||||||
|
|
@ -320,6 +373,7 @@ message ListTaintsResponse {
|
||||||
uint64 match_count = 2;
|
uint64 match_count = 2;
|
||||||
uint64 page = 3;
|
uint64 page = 3;
|
||||||
uint64 page_size = 4;
|
uint64 page_size = 4;
|
||||||
|
RelatedEntities index = 5;
|
||||||
}
|
}
|
||||||
|
|
||||||
message ListPartitionsRequest {
|
message ListPartitionsRequest {
|
||||||
|
|
@ -332,6 +386,7 @@ message ListPartitionsResponse {
|
||||||
uint64 match_count = 2;
|
uint64 match_count = 2;
|
||||||
uint64 page = 3;
|
uint64 page = 3;
|
||||||
uint64 page_size = 4;
|
uint64 page_size = 4;
|
||||||
|
RelatedEntities index = 5;
|
||||||
}
|
}
|
||||||
|
|
||||||
message ListJobRunsRequest {
|
message ListJobRunsRequest {
|
||||||
|
|
@ -344,15 +399,16 @@ message ListJobRunsResponse {
|
||||||
uint64 match_count = 2;
|
uint64 match_count = 2;
|
||||||
uint64 page = 3;
|
uint64 page = 3;
|
||||||
uint64 page_size = 4;
|
uint64 page_size = 4;
|
||||||
|
RelatedEntities index = 5;
|
||||||
}
|
}
|
||||||
|
|
||||||
message CreateWantRequest {
|
message CreateWantRequest {
|
||||||
repeated PartitionRef partitions = 1;
|
repeated PartitionRef partitions = 1;
|
||||||
|
// User-created wants are always originating (have explicit freshness requirements)
|
||||||
uint64 data_timestamp = 2;
|
uint64 data_timestamp = 2;
|
||||||
uint64 ttl_seconds = 3;
|
uint64 ttl_seconds = 3;
|
||||||
uint64 sla_seconds = 4;
|
uint64 sla_seconds = 4;
|
||||||
EventSource source = 5;
|
optional string comment = 5;
|
||||||
optional string comment = 6;
|
|
||||||
}
|
}
|
||||||
message CreateWantResponse {
|
message CreateWantResponse {
|
||||||
WantDetail data = 1;
|
WantDetail data = 1;
|
||||||
|
|
@ -372,6 +428,23 @@ message GetWantRequest {
|
||||||
}
|
}
|
||||||
message GetWantResponse {
|
message GetWantResponse {
|
||||||
WantDetail data = 1;
|
WantDetail data = 1;
|
||||||
|
RelatedEntities index = 2;
|
||||||
|
}
|
||||||
|
|
||||||
|
message GetPartitionRequest {
|
||||||
|
string partition_ref = 1;
|
||||||
|
}
|
||||||
|
message GetPartitionResponse {
|
||||||
|
PartitionDetail data = 1;
|
||||||
|
RelatedEntities index = 2;
|
||||||
|
}
|
||||||
|
|
||||||
|
message GetJobRunRequest {
|
||||||
|
string job_run_id = 1;
|
||||||
|
}
|
||||||
|
message GetJobRunResponse {
|
||||||
|
JobRunDetail data = 1;
|
||||||
|
RelatedEntities index = 2;
|
||||||
}
|
}
|
||||||
|
|
||||||
message CreateTaintRequest {
|
message CreateTaintRequest {
|
||||||
|
|
|
||||||
|
|
@ -1,14 +1,15 @@
|
||||||
use crate::PartitionStatusCode::{PartitionFailed, PartitionLive};
|
use crate::PartitionStatusCode::{PartitionFailed, PartitionLive};
|
||||||
use crate::data_build_event::Event;
|
use crate::data_build_event::Event;
|
||||||
use crate::job_run_state::{JobInfo, JobRunWithState, QueuedState};
|
use crate::job_run_state::{JobInfo, JobRunWithState, QueuedState, TimingInfo};
|
||||||
use crate::util::current_timestamp;
|
use crate::util::current_timestamp;
|
||||||
|
use crate::want_create_event_v1::Lifetime;
|
||||||
use crate::{
|
use crate::{
|
||||||
CancelWantRequest, CancelWantResponse, CreateTaintRequest, CreateTaintResponse,
|
CancelWantRequest, CancelWantResponse, CreateTaintRequest, CreateTaintResponse,
|
||||||
CreateWantRequest, CreateWantResponse, EventSource, GetWantResponse, JobRunBufferEventV1,
|
CreateWantRequest, CreateWantResponse, EventSource, GetWantResponse, JobRunBufferEventV1,
|
||||||
JobRunDetail, JobRunStatus, JobRunStatusCode, JobTriggeredEvent, ManuallyTriggeredEvent,
|
JobRunDetail, JobRunStatus, JobRunStatusCode, JobTriggeredEvent, ManuallyTriggeredEvent,
|
||||||
PartitionDetail, PartitionRef, PartitionStatus, PartitionStatusCode, TaintCancelEventV1,
|
OriginatingLifetime, PartitionDetail, PartitionRef, PartitionStatus, PartitionStatusCode,
|
||||||
TaintCreateEventV1, TaintDetail, WantAttributedPartitions, WantCancelEventV1,
|
TaintCancelEventV1, TaintCreateEventV1, TaintDetail, WantAttributedPartitions,
|
||||||
WantCreateEventV1, WantDetail, WantStatus, WantStatusCode, event_source,
|
WantCancelEventV1, WantCreateEventV1, WantDetail, WantStatus, WantStatusCode, event_source,
|
||||||
};
|
};
|
||||||
use uuid::Uuid;
|
use uuid::Uuid;
|
||||||
|
|
||||||
|
|
@ -19,17 +20,23 @@ impl From<&WantCreateEventV1> for WantDetail {
|
||||||
}
|
}
|
||||||
impl From<WantCreateEventV1> for WantDetail {
|
impl From<WantCreateEventV1> for WantDetail {
|
||||||
fn from(e: WantCreateEventV1) -> Self {
|
fn from(e: WantCreateEventV1) -> Self {
|
||||||
|
// Convert want_create_event_v1::Lifetime to want_detail::Lifetime
|
||||||
|
let lifetime = e.lifetime.map(|l| match l {
|
||||||
|
Lifetime::Originating(orig) => crate::want_detail::Lifetime::Originating(orig),
|
||||||
|
Lifetime::Ephemeral(eph) => crate::want_detail::Lifetime::Ephemeral(eph),
|
||||||
|
});
|
||||||
|
|
||||||
WantDetail {
|
WantDetail {
|
||||||
want_id: e.want_id,
|
want_id: e.want_id,
|
||||||
partitions: e.partitions,
|
partitions: e.partitions,
|
||||||
upstreams: vec![],
|
upstreams: vec![],
|
||||||
data_timestamp: e.data_timestamp,
|
lifetime,
|
||||||
ttl_seconds: e.ttl_seconds,
|
|
||||||
sla_seconds: e.sla_seconds,
|
|
||||||
source: e.source,
|
|
||||||
comment: e.comment,
|
comment: e.comment,
|
||||||
status: Some(WantStatusCode::WantIdle.into()),
|
status: Some(WantStatusCode::WantIdle.into()),
|
||||||
last_updated_timestamp: current_timestamp(),
|
last_updated_timestamp: current_timestamp(),
|
||||||
|
job_run_ids: vec![],
|
||||||
|
derivative_want_ids: vec![],
|
||||||
|
job_runs: vec![],
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -74,27 +81,39 @@ impl From<WantStatusCode> for WantStatus {
|
||||||
|
|
||||||
impl From<JobRunBufferEventV1> for JobRunDetail {
|
impl From<JobRunBufferEventV1> for JobRunDetail {
|
||||||
fn from(value: JobRunBufferEventV1) -> Self {
|
fn from(value: JobRunBufferEventV1) -> Self {
|
||||||
|
use std::collections::HashMap;
|
||||||
Self {
|
Self {
|
||||||
id: value.job_run_id,
|
id: value.job_run_id,
|
||||||
|
job_label: value.job_label,
|
||||||
status: Some(JobRunStatusCode::JobRunQueued.into()),
|
status: Some(JobRunStatusCode::JobRunQueued.into()),
|
||||||
last_heartbeat_at: None,
|
last_heartbeat_at: None,
|
||||||
building_partitions: value.building_partitions,
|
building_partitions: value.building_partitions,
|
||||||
servicing_wants: value.want_attributed_partitions,
|
servicing_wants: value.want_attributed_partitions,
|
||||||
|
read_deps: vec![],
|
||||||
|
read_partition_uuids: HashMap::new(),
|
||||||
|
wrote_partition_uuids: HashMap::new(),
|
||||||
|
derivative_want_ids: vec![],
|
||||||
|
queued_at: Some(current_timestamp()),
|
||||||
|
started_at: None,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
impl From<JobRunBufferEventV1> for JobRunWithState<QueuedState> {
|
impl From<JobRunBufferEventV1> for JobRunWithState<QueuedState> {
|
||||||
fn from(event: JobRunBufferEventV1) -> Self {
|
fn from(event: JobRunBufferEventV1) -> Self {
|
||||||
|
let queued_at = current_timestamp();
|
||||||
JobRunWithState {
|
JobRunWithState {
|
||||||
info: JobInfo {
|
info: JobInfo {
|
||||||
id: event.job_run_id,
|
id: event.job_run_id,
|
||||||
|
job_label: event.job_label,
|
||||||
building_partitions: event.building_partitions,
|
building_partitions: event.building_partitions,
|
||||||
servicing_wants: event.want_attributed_partitions,
|
servicing_wants: event.want_attributed_partitions,
|
||||||
},
|
},
|
||||||
state: QueuedState {
|
timing: TimingInfo {
|
||||||
queued_at: current_timestamp(),
|
queued_at,
|
||||||
|
started_at: None,
|
||||||
},
|
},
|
||||||
|
state: QueuedState { queued_at },
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -190,13 +209,15 @@ impl From<&WantDetail> for WantAttributedPartitions {
|
||||||
|
|
||||||
impl From<CreateWantRequest> for WantCreateEventV1 {
|
impl From<CreateWantRequest> for WantCreateEventV1 {
|
||||||
fn from(value: CreateWantRequest) -> Self {
|
fn from(value: CreateWantRequest) -> Self {
|
||||||
|
// User-created wants are always originating (have explicit freshness requirements)
|
||||||
WantCreateEventV1 {
|
WantCreateEventV1 {
|
||||||
want_id: Uuid::new_v4().into(),
|
want_id: Uuid::new_v4().into(),
|
||||||
partitions: value.partitions,
|
partitions: value.partitions,
|
||||||
data_timestamp: value.data_timestamp,
|
lifetime: Some(Lifetime::Originating(OriginatingLifetime {
|
||||||
ttl_seconds: value.ttl_seconds,
|
data_timestamp: value.data_timestamp,
|
||||||
sla_seconds: value.sla_seconds,
|
ttl_seconds: value.ttl_seconds,
|
||||||
source: value.source,
|
sla_seconds: value.sla_seconds,
|
||||||
|
})),
|
||||||
comment: value.comment,
|
comment: value.comment,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -210,7 +231,10 @@ impl Into<CreateWantResponse> for Option<WantDetail> {
|
||||||
|
|
||||||
impl Into<GetWantResponse> for Option<WantDetail> {
|
impl Into<GetWantResponse> for Option<WantDetail> {
|
||||||
fn into(self) -> GetWantResponse {
|
fn into(self) -> GetWantResponse {
|
||||||
GetWantResponse { data: self }
|
GetWantResponse {
|
||||||
|
data: self,
|
||||||
|
index: None,
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -2,7 +2,7 @@ use crate::build_event_log::BELStorage;
|
||||||
use crate::build_state::BuildState;
|
use crate::build_state::BuildState;
|
||||||
use crate::commands::Command;
|
use crate::commands::Command;
|
||||||
use crate::web::templates::{
|
use crate::web::templates::{
|
||||||
BaseContext, HomePage, JobRunDetailPage, JobRunDetailView, JobRunsListPage,
|
BaseContext, DerivativeWantView, HomePage, JobRunDetailPage, JobRunDetailView, JobRunsListPage,
|
||||||
PartitionDetailPage, PartitionDetailView, PartitionsListPage, WantCreatePage, WantDetailPage,
|
PartitionDetailPage, PartitionDetailView, PartitionsListPage, WantCreatePage, WantDetailPage,
|
||||||
WantDetailView, WantsListPage,
|
WantDetailView, WantsListPage,
|
||||||
};
|
};
|
||||||
|
|
@ -260,9 +260,17 @@ async fn want_detail_page(
|
||||||
|
|
||||||
match build_state.get_want(&want_id) {
|
match build_state.get_want(&want_id) {
|
||||||
Some(want) => {
|
Some(want) => {
|
||||||
|
// Fetch derivative wants
|
||||||
|
let derivative_wants: Vec<_> = want
|
||||||
|
.derivative_want_ids
|
||||||
|
.iter()
|
||||||
|
.filter_map(|id| build_state.get_want(id))
|
||||||
|
.map(|w| DerivativeWantView::from(&w))
|
||||||
|
.collect();
|
||||||
|
|
||||||
let template = WantDetailPage {
|
let template = WantDetailPage {
|
||||||
base: BaseContext::default(),
|
base: BaseContext::default(),
|
||||||
want: WantDetailView::from(want),
|
want: WantDetailView::new(&want, derivative_wants),
|
||||||
};
|
};
|
||||||
match template.render() {
|
match template.render() {
|
||||||
Ok(html) => Html(html).into_response(),
|
Ok(html) => Html(html).into_response(),
|
||||||
|
|
@ -433,7 +441,7 @@ async fn list_wants_json(
|
||||||
.into_response();
|
.into_response();
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
let response = build_state.list_wants(¶ms);
|
let response = build_state.list_wants_with_index(¶ms);
|
||||||
|
|
||||||
(StatusCode::OK, Json(response)).into_response()
|
(StatusCode::OK, Json(response)).into_response()
|
||||||
}
|
}
|
||||||
|
|
@ -456,10 +464,8 @@ async fn get_want_json(
|
||||||
.into_response();
|
.into_response();
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
let response = build_state.get_want(&want_id);
|
match build_state.get_want_with_index(&want_id) {
|
||||||
|
Some(response) => (StatusCode::OK, Json(response)).into_response(),
|
||||||
match response {
|
|
||||||
Some(want) => (StatusCode::OK, Json(GetWantResponse { data: Some(want) })).into_response(),
|
|
||||||
None => {
|
None => {
|
||||||
tracing::debug!("Want not found: {}", want_id);
|
tracing::debug!("Want not found: {}", want_id);
|
||||||
(
|
(
|
||||||
|
|
@ -608,7 +614,7 @@ async fn list_partitions_json(
|
||||||
.into_response();
|
.into_response();
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
let response = build_state.list_partitions(¶ms);
|
let response = build_state.list_partitions_with_index(¶ms);
|
||||||
|
|
||||||
(StatusCode::OK, Json(response)).into_response()
|
(StatusCode::OK, Json(response)).into_response()
|
||||||
}
|
}
|
||||||
|
|
@ -631,7 +637,7 @@ async fn list_job_runs_json(
|
||||||
.into_response();
|
.into_response();
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
let response = build_state.list_job_runs(¶ms);
|
let response = build_state.list_job_runs_with_index(¶ms);
|
||||||
|
|
||||||
(StatusCode::OK, Json(response)).into_response()
|
(StatusCode::OK, Json(response)).into_response()
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -256,6 +256,7 @@ impl SubProcessCompleted {
|
||||||
pub fn to_event(&self, job_run_id: &Uuid) -> Event {
|
pub fn to_event(&self, job_run_id: &Uuid) -> Event {
|
||||||
Event::JobRunSuccessV1(JobRunSuccessEventV1 {
|
Event::JobRunSuccessV1(JobRunSuccessEventV1 {
|
||||||
job_run_id: job_run_id.to_string(),
|
job_run_id: job_run_id.to_string(),
|
||||||
|
read_deps: self.read_deps.clone(),
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -394,6 +395,7 @@ impl ToEvent for SubProcessCompleted {
|
||||||
fn to_event(&self, job_run_id: &Uuid) -> Event {
|
fn to_event(&self, job_run_id: &Uuid) -> Event {
|
||||||
Event::JobRunSuccessV1(JobRunSuccessEventV1 {
|
Event::JobRunSuccessV1(JobRunSuccessEventV1 {
|
||||||
job_run_id: job_run_id.to_string(),
|
job_run_id: job_run_id.to_string(),
|
||||||
|
read_deps: self.read_deps.clone(),
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -1,9 +1,11 @@
|
||||||
use crate::partition_state::{BuildingPartitionRef, FailedPartitionRef, LivePartitionRef};
|
use crate::partition_state::{BuildingPartitionRef, FailedPartitionRef, LivePartitionRef};
|
||||||
use crate::util::current_timestamp;
|
use crate::util::{HasRelatedIds, RelatedIds, current_timestamp};
|
||||||
use crate::{
|
use crate::{
|
||||||
EventSource, JobRunDetail, JobRunStatusCode, MissingDeps, PartitionRef, ReadDeps,
|
EventSource, JobRunDetail, JobRunStatusCode, MissingDeps, PartitionRef, ReadDeps,
|
||||||
WantAttributedPartitions,
|
WantAttributedPartitions,
|
||||||
};
|
};
|
||||||
|
use std::collections::BTreeMap;
|
||||||
|
use uuid::Uuid;
|
||||||
|
|
||||||
/// State: Job has been queued but not yet started
|
/// State: Job has been queued but not yet started
|
||||||
#[derive(Debug, Clone)]
|
#[derive(Debug, Clone)]
|
||||||
|
|
@ -22,6 +24,12 @@ pub struct RunningState {
|
||||||
#[derive(Debug, Clone)]
|
#[derive(Debug, Clone)]
|
||||||
pub struct SucceededState {
|
pub struct SucceededState {
|
||||||
pub completed_at: u64,
|
pub completed_at: u64,
|
||||||
|
/// The read dependencies reported by the job, preserving impacted→read relationships
|
||||||
|
pub read_deps: Vec<ReadDeps>,
|
||||||
|
/// Resolved UUIDs for partitions that were read (ref → UUID at read time)
|
||||||
|
pub read_partition_uuids: BTreeMap<String, String>,
|
||||||
|
/// Resolved UUIDs for partitions that were written (ref → UUID)
|
||||||
|
pub wrote_partition_uuids: BTreeMap<String, String>,
|
||||||
}
|
}
|
||||||
|
|
||||||
/// State: Job failed during execution
|
/// State: Job failed during execution
|
||||||
|
|
@ -37,6 +45,8 @@ pub struct DepMissState {
|
||||||
pub detected_at: u64,
|
pub detected_at: u64,
|
||||||
pub missing_deps: Vec<MissingDeps>,
|
pub missing_deps: Vec<MissingDeps>,
|
||||||
pub read_deps: Vec<ReadDeps>,
|
pub read_deps: Vec<ReadDeps>,
|
||||||
|
/// Want IDs of ephemeral wants spawned by this dep-miss
|
||||||
|
pub derivative_want_ids: Vec<String>,
|
||||||
}
|
}
|
||||||
|
|
||||||
/// State: Job was explicitly canceled
|
/// State: Job was explicitly canceled
|
||||||
|
|
@ -51,14 +61,25 @@ pub struct CanceledState {
|
||||||
#[derive(Debug, Clone)]
|
#[derive(Debug, Clone)]
|
||||||
pub struct JobInfo {
|
pub struct JobInfo {
|
||||||
pub id: String,
|
pub id: String,
|
||||||
|
pub job_label: String,
|
||||||
pub building_partitions: Vec<PartitionRef>,
|
pub building_partitions: Vec<PartitionRef>,
|
||||||
pub servicing_wants: Vec<WantAttributedPartitions>,
|
pub servicing_wants: Vec<WantAttributedPartitions>,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Timing information preserved across state transitions
|
||||||
|
#[derive(Debug, Clone)]
|
||||||
|
pub struct TimingInfo {
|
||||||
|
/// When the job was first queued
|
||||||
|
pub queued_at: u64,
|
||||||
|
/// When the job started running (None if still queued or canceled before starting)
|
||||||
|
pub started_at: Option<u64>,
|
||||||
|
}
|
||||||
|
|
||||||
/// Generic job run struct parameterized by state
|
/// Generic job run struct parameterized by state
|
||||||
#[derive(Debug, Clone)]
|
#[derive(Debug, Clone)]
|
||||||
pub struct JobRunWithState<S> {
|
pub struct JobRunWithState<S> {
|
||||||
pub info: JobInfo,
|
pub info: JobInfo,
|
||||||
|
pub timing: TimingInfo,
|
||||||
pub state: S,
|
pub state: S,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -80,6 +101,10 @@ impl JobRunWithState<QueuedState> {
|
||||||
pub fn start_running(self, timestamp: u64) -> JobRunWithState<RunningState> {
|
pub fn start_running(self, timestamp: u64) -> JobRunWithState<RunningState> {
|
||||||
JobRunWithState {
|
JobRunWithState {
|
||||||
info: self.info,
|
info: self.info,
|
||||||
|
timing: TimingInfo {
|
||||||
|
queued_at: self.timing.queued_at,
|
||||||
|
started_at: Some(timestamp),
|
||||||
|
},
|
||||||
state: RunningState {
|
state: RunningState {
|
||||||
started_at: timestamp,
|
started_at: timestamp,
|
||||||
last_heartbeat_at: timestamp, // Initialize to start time
|
last_heartbeat_at: timestamp, // Initialize to start time
|
||||||
|
|
@ -96,6 +121,7 @@ impl JobRunWithState<QueuedState> {
|
||||||
) -> JobRunWithState<CanceledState> {
|
) -> JobRunWithState<CanceledState> {
|
||||||
JobRunWithState {
|
JobRunWithState {
|
||||||
info: self.info,
|
info: self.info,
|
||||||
|
timing: self.timing, // Preserve timing (started_at remains None)
|
||||||
state: CanceledState {
|
state: CanceledState {
|
||||||
canceled_at: timestamp,
|
canceled_at: timestamp,
|
||||||
source,
|
source,
|
||||||
|
|
@ -113,11 +139,21 @@ impl JobRunWithState<RunningState> {
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Transition from Running to Succeeded
|
/// Transition from Running to Succeeded
|
||||||
pub fn succeed(self, timestamp: u64) -> JobRunWithState<SucceededState> {
|
pub fn succeed(
|
||||||
|
self,
|
||||||
|
timestamp: u64,
|
||||||
|
read_deps: Vec<ReadDeps>,
|
||||||
|
read_partition_uuids: BTreeMap<String, String>,
|
||||||
|
wrote_partition_uuids: BTreeMap<String, String>,
|
||||||
|
) -> JobRunWithState<SucceededState> {
|
||||||
JobRunWithState {
|
JobRunWithState {
|
||||||
info: self.info,
|
info: self.info,
|
||||||
|
timing: self.timing,
|
||||||
state: SucceededState {
|
state: SucceededState {
|
||||||
completed_at: timestamp,
|
completed_at: timestamp,
|
||||||
|
read_deps,
|
||||||
|
read_partition_uuids,
|
||||||
|
wrote_partition_uuids,
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -126,6 +162,7 @@ impl JobRunWithState<RunningState> {
|
||||||
pub fn fail(self, timestamp: u64, reason: String) -> JobRunWithState<FailedState> {
|
pub fn fail(self, timestamp: u64, reason: String) -> JobRunWithState<FailedState> {
|
||||||
JobRunWithState {
|
JobRunWithState {
|
||||||
info: self.info,
|
info: self.info,
|
||||||
|
timing: self.timing,
|
||||||
state: FailedState {
|
state: FailedState {
|
||||||
failed_at: timestamp,
|
failed_at: timestamp,
|
||||||
failure_reason: reason,
|
failure_reason: reason,
|
||||||
|
|
@ -141,11 +178,13 @@ impl JobRunWithState<RunningState> {
|
||||||
read_deps: Vec<ReadDeps>,
|
read_deps: Vec<ReadDeps>,
|
||||||
) -> JobRunWithState<DepMissState> {
|
) -> JobRunWithState<DepMissState> {
|
||||||
JobRunWithState {
|
JobRunWithState {
|
||||||
|
timing: self.timing,
|
||||||
info: self.info,
|
info: self.info,
|
||||||
state: DepMissState {
|
state: DepMissState {
|
||||||
detected_at: timestamp,
|
detected_at: timestamp,
|
||||||
missing_deps,
|
missing_deps,
|
||||||
read_deps,
|
read_deps,
|
||||||
|
derivative_want_ids: vec![], // Populated later when ephemeral wants are created
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -159,6 +198,7 @@ impl JobRunWithState<RunningState> {
|
||||||
) -> JobRunWithState<CanceledState> {
|
) -> JobRunWithState<CanceledState> {
|
||||||
JobRunWithState {
|
JobRunWithState {
|
||||||
info: self.info,
|
info: self.info,
|
||||||
|
timing: self.timing,
|
||||||
state: CanceledState {
|
state: CanceledState {
|
||||||
canceled_at: timestamp,
|
canceled_at: timestamp,
|
||||||
source,
|
source,
|
||||||
|
|
@ -231,6 +271,21 @@ impl JobRunWithState<SucceededState> {
|
||||||
.map(|p| LivePartitionRef(p.clone()))
|
.map(|p| LivePartitionRef(p.clone()))
|
||||||
.collect()
|
.collect()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Get the read dependencies reported by the job
|
||||||
|
pub fn get_read_deps(&self) -> &[ReadDeps] {
|
||||||
|
&self.state.read_deps
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Get the resolved UUIDs for partitions that were read
|
||||||
|
pub fn get_read_partition_uuids(&self) -> &BTreeMap<String, String> {
|
||||||
|
&self.state.read_partition_uuids
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Get the resolved UUIDs for partitions that were written
|
||||||
|
pub fn get_wrote_partition_uuids(&self) -> &BTreeMap<String, String> {
|
||||||
|
&self.state.wrote_partition_uuids
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
impl JobRunWithState<FailedState> {
|
impl JobRunWithState<FailedState> {
|
||||||
|
|
@ -273,6 +328,17 @@ impl JobRunWithState<DepMissState> {
|
||||||
pub fn get_read_deps(&self) -> &[ReadDeps] {
|
pub fn get_read_deps(&self) -> &[ReadDeps] {
|
||||||
&self.state.read_deps
|
&self.state.read_deps
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Add a derivative want ID (ephemeral want spawned by this dep-miss)
|
||||||
|
pub fn add_derivative_want_id(&mut self, want_id: &str) {
|
||||||
|
if !self
|
||||||
|
.state
|
||||||
|
.derivative_want_ids
|
||||||
|
.contains(&want_id.to_string())
|
||||||
|
{
|
||||||
|
self.state.derivative_want_ids.push(want_id.to_string());
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
impl JobRunWithState<CanceledState> {
|
impl JobRunWithState<CanceledState> {
|
||||||
|
|
@ -290,52 +356,222 @@ impl JobRunWithState<CanceledState> {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ==================== HasRelatedIds trait implementation ====================
|
||||||
|
|
||||||
|
impl HasRelatedIds for JobRun {
|
||||||
|
/// Get the IDs of all entities this job run references.
|
||||||
|
/// Note: derivative_want_ids come from BuildState, not from JobRun itself.
|
||||||
|
fn related_ids(&self) -> RelatedIds {
|
||||||
|
// Partition refs from building_partitions (all states have this)
|
||||||
|
let partition_refs: Vec<String> = match self {
|
||||||
|
JobRun::Queued(jr) => jr
|
||||||
|
.info
|
||||||
|
.building_partitions
|
||||||
|
.iter()
|
||||||
|
.map(|p| p.r#ref.clone())
|
||||||
|
.collect(),
|
||||||
|
JobRun::Running(jr) => jr
|
||||||
|
.info
|
||||||
|
.building_partitions
|
||||||
|
.iter()
|
||||||
|
.map(|p| p.r#ref.clone())
|
||||||
|
.collect(),
|
||||||
|
JobRun::Succeeded(jr) => jr
|
||||||
|
.info
|
||||||
|
.building_partitions
|
||||||
|
.iter()
|
||||||
|
.map(|p| p.r#ref.clone())
|
||||||
|
.collect(),
|
||||||
|
JobRun::Failed(jr) => jr
|
||||||
|
.info
|
||||||
|
.building_partitions
|
||||||
|
.iter()
|
||||||
|
.map(|p| p.r#ref.clone())
|
||||||
|
.collect(),
|
||||||
|
JobRun::DepMiss(jr) => jr
|
||||||
|
.info
|
||||||
|
.building_partitions
|
||||||
|
.iter()
|
||||||
|
.map(|p| p.r#ref.clone())
|
||||||
|
.collect(),
|
||||||
|
JobRun::Canceled(jr) => jr
|
||||||
|
.info
|
||||||
|
.building_partitions
|
||||||
|
.iter()
|
||||||
|
.map(|p| p.r#ref.clone())
|
||||||
|
.collect(),
|
||||||
|
};
|
||||||
|
|
||||||
|
// Partition UUIDs from read/write lineage (only Succeeded state has these)
|
||||||
|
let partition_uuids: Vec<Uuid> = match self {
|
||||||
|
JobRun::Succeeded(jr) => {
|
||||||
|
let mut uuids = Vec::new();
|
||||||
|
for uuid_str in jr.state.read_partition_uuids.values() {
|
||||||
|
if let Ok(uuid) = Uuid::parse_str(uuid_str) {
|
||||||
|
uuids.push(uuid);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for uuid_str in jr.state.wrote_partition_uuids.values() {
|
||||||
|
if let Ok(uuid) = Uuid::parse_str(uuid_str) {
|
||||||
|
if !uuids.contains(&uuid) {
|
||||||
|
uuids.push(uuid);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
uuids
|
||||||
|
}
|
||||||
|
_ => vec![],
|
||||||
|
};
|
||||||
|
|
||||||
|
// Want IDs from servicing_wants (all states have this)
|
||||||
|
let want_ids: Vec<String> = match self {
|
||||||
|
JobRun::Queued(jr) => jr
|
||||||
|
.info
|
||||||
|
.servicing_wants
|
||||||
|
.iter()
|
||||||
|
.map(|w| w.want_id.clone())
|
||||||
|
.collect(),
|
||||||
|
JobRun::Running(jr) => jr
|
||||||
|
.info
|
||||||
|
.servicing_wants
|
||||||
|
.iter()
|
||||||
|
.map(|w| w.want_id.clone())
|
||||||
|
.collect(),
|
||||||
|
JobRun::Succeeded(jr) => jr
|
||||||
|
.info
|
||||||
|
.servicing_wants
|
||||||
|
.iter()
|
||||||
|
.map(|w| w.want_id.clone())
|
||||||
|
.collect(),
|
||||||
|
JobRun::Failed(jr) => jr
|
||||||
|
.info
|
||||||
|
.servicing_wants
|
||||||
|
.iter()
|
||||||
|
.map(|w| w.want_id.clone())
|
||||||
|
.collect(),
|
||||||
|
JobRun::DepMiss(jr) => jr
|
||||||
|
.info
|
||||||
|
.servicing_wants
|
||||||
|
.iter()
|
||||||
|
.map(|w| w.want_id.clone())
|
||||||
|
.collect(),
|
||||||
|
JobRun::Canceled(jr) => jr
|
||||||
|
.info
|
||||||
|
.servicing_wants
|
||||||
|
.iter()
|
||||||
|
.map(|w| w.want_id.clone())
|
||||||
|
.collect(),
|
||||||
|
};
|
||||||
|
|
||||||
|
RelatedIds {
|
||||||
|
partition_refs,
|
||||||
|
partition_uuids,
|
||||||
|
job_run_ids: vec![],
|
||||||
|
want_ids,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// ==================== Conversion to JobRunDetail for API ====================
|
// ==================== Conversion to JobRunDetail for API ====================
|
||||||
|
|
||||||
impl JobRun {
|
impl JobRun {
|
||||||
pub fn to_detail(&self) -> JobRunDetail {
|
pub fn to_detail(&self) -> JobRunDetail {
|
||||||
|
use std::collections::HashMap;
|
||||||
|
|
||||||
match self {
|
match self {
|
||||||
JobRun::Queued(queued) => JobRunDetail {
|
JobRun::Queued(queued) => JobRunDetail {
|
||||||
id: queued.info.id.clone(),
|
id: queued.info.id.clone(),
|
||||||
|
job_label: queued.info.job_label.clone(),
|
||||||
status: Some(JobRunStatusCode::JobRunQueued.into()),
|
status: Some(JobRunStatusCode::JobRunQueued.into()),
|
||||||
last_heartbeat_at: None,
|
last_heartbeat_at: None,
|
||||||
building_partitions: queued.info.building_partitions.clone(),
|
building_partitions: queued.info.building_partitions.clone(),
|
||||||
servicing_wants: queued.info.servicing_wants.clone(),
|
servicing_wants: queued.info.servicing_wants.clone(),
|
||||||
|
read_deps: vec![],
|
||||||
|
read_partition_uuids: HashMap::new(),
|
||||||
|
wrote_partition_uuids: HashMap::new(),
|
||||||
|
derivative_want_ids: vec![],
|
||||||
|
queued_at: Some(queued.timing.queued_at),
|
||||||
|
started_at: queued.timing.started_at,
|
||||||
},
|
},
|
||||||
JobRun::Running(running) => JobRunDetail {
|
JobRun::Running(running) => JobRunDetail {
|
||||||
id: running.info.id.clone(),
|
id: running.info.id.clone(),
|
||||||
|
job_label: running.info.job_label.clone(),
|
||||||
status: Some(JobRunStatusCode::JobRunRunning.into()),
|
status: Some(JobRunStatusCode::JobRunRunning.into()),
|
||||||
last_heartbeat_at: Some(running.state.last_heartbeat_at),
|
last_heartbeat_at: Some(running.state.last_heartbeat_at),
|
||||||
building_partitions: running.info.building_partitions.clone(),
|
building_partitions: running.info.building_partitions.clone(),
|
||||||
servicing_wants: running.info.servicing_wants.clone(),
|
servicing_wants: running.info.servicing_wants.clone(),
|
||||||
|
read_deps: vec![],
|
||||||
|
read_partition_uuids: HashMap::new(),
|
||||||
|
wrote_partition_uuids: HashMap::new(),
|
||||||
|
derivative_want_ids: vec![],
|
||||||
|
queued_at: Some(running.timing.queued_at),
|
||||||
|
started_at: running.timing.started_at,
|
||||||
},
|
},
|
||||||
JobRun::Succeeded(succeeded) => JobRunDetail {
|
JobRun::Succeeded(succeeded) => JobRunDetail {
|
||||||
id: succeeded.info.id.clone(),
|
id: succeeded.info.id.clone(),
|
||||||
|
job_label: succeeded.info.job_label.clone(),
|
||||||
status: Some(JobRunStatusCode::JobRunSucceeded.into()),
|
status: Some(JobRunStatusCode::JobRunSucceeded.into()),
|
||||||
last_heartbeat_at: None,
|
last_heartbeat_at: None,
|
||||||
building_partitions: succeeded.info.building_partitions.clone(),
|
building_partitions: succeeded.info.building_partitions.clone(),
|
||||||
servicing_wants: succeeded.info.servicing_wants.clone(),
|
servicing_wants: succeeded.info.servicing_wants.clone(),
|
||||||
|
read_deps: succeeded.state.read_deps.clone(),
|
||||||
|
read_partition_uuids: succeeded
|
||||||
|
.state
|
||||||
|
.read_partition_uuids
|
||||||
|
.clone()
|
||||||
|
.into_iter()
|
||||||
|
.collect(),
|
||||||
|
wrote_partition_uuids: succeeded
|
||||||
|
.state
|
||||||
|
.wrote_partition_uuids
|
||||||
|
.clone()
|
||||||
|
.into_iter()
|
||||||
|
.collect(),
|
||||||
|
derivative_want_ids: vec![],
|
||||||
|
queued_at: Some(succeeded.timing.queued_at),
|
||||||
|
started_at: succeeded.timing.started_at,
|
||||||
},
|
},
|
||||||
JobRun::Failed(failed) => JobRunDetail {
|
JobRun::Failed(failed) => JobRunDetail {
|
||||||
id: failed.info.id.clone(),
|
id: failed.info.id.clone(),
|
||||||
|
job_label: failed.info.job_label.clone(),
|
||||||
status: Some(JobRunStatusCode::JobRunFailed.into()),
|
status: Some(JobRunStatusCode::JobRunFailed.into()),
|
||||||
last_heartbeat_at: None,
|
last_heartbeat_at: None,
|
||||||
building_partitions: failed.info.building_partitions.clone(),
|
building_partitions: failed.info.building_partitions.clone(),
|
||||||
servicing_wants: failed.info.servicing_wants.clone(),
|
servicing_wants: failed.info.servicing_wants.clone(),
|
||||||
|
read_deps: vec![],
|
||||||
|
read_partition_uuids: HashMap::new(),
|
||||||
|
wrote_partition_uuids: HashMap::new(),
|
||||||
|
derivative_want_ids: vec![],
|
||||||
|
queued_at: Some(failed.timing.queued_at),
|
||||||
|
started_at: failed.timing.started_at,
|
||||||
},
|
},
|
||||||
JobRun::DepMiss(dep_miss) => JobRunDetail {
|
JobRun::DepMiss(dep_miss) => JobRunDetail {
|
||||||
id: dep_miss.info.id.clone(),
|
id: dep_miss.info.id.clone(),
|
||||||
|
job_label: dep_miss.info.job_label.clone(),
|
||||||
status: Some(JobRunStatusCode::JobRunDepMiss.into()),
|
status: Some(JobRunStatusCode::JobRunDepMiss.into()),
|
||||||
last_heartbeat_at: None,
|
last_heartbeat_at: None,
|
||||||
building_partitions: dep_miss.info.building_partitions.clone(),
|
building_partitions: dep_miss.info.building_partitions.clone(),
|
||||||
servicing_wants: dep_miss.info.servicing_wants.clone(),
|
servicing_wants: dep_miss.info.servicing_wants.clone(),
|
||||||
|
read_deps: dep_miss.state.read_deps.clone(),
|
||||||
|
read_partition_uuids: HashMap::new(),
|
||||||
|
wrote_partition_uuids: HashMap::new(),
|
||||||
|
derivative_want_ids: dep_miss.state.derivative_want_ids.clone(),
|
||||||
|
queued_at: Some(dep_miss.timing.queued_at),
|
||||||
|
started_at: dep_miss.timing.started_at,
|
||||||
},
|
},
|
||||||
JobRun::Canceled(canceled) => JobRunDetail {
|
JobRun::Canceled(canceled) => JobRunDetail {
|
||||||
id: canceled.info.id.clone(),
|
id: canceled.info.id.clone(),
|
||||||
|
job_label: canceled.info.job_label.clone(),
|
||||||
status: Some(JobRunStatusCode::JobRunCanceled.into()),
|
status: Some(JobRunStatusCode::JobRunCanceled.into()),
|
||||||
last_heartbeat_at: None,
|
last_heartbeat_at: None,
|
||||||
building_partitions: canceled.info.building_partitions.clone(),
|
building_partitions: canceled.info.building_partitions.clone(),
|
||||||
servicing_wants: canceled.info.servicing_wants.clone(),
|
servicing_wants: canceled.info.servicing_wants.clone(),
|
||||||
|
read_deps: vec![],
|
||||||
|
read_partition_uuids: HashMap::new(),
|
||||||
|
wrote_partition_uuids: HashMap::new(),
|
||||||
|
derivative_want_ids: vec![],
|
||||||
|
queued_at: Some(canceled.timing.queued_at),
|
||||||
|
started_at: canceled.timing.started_at,
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -459,12 +459,13 @@ impl<S: BELStorage + Debug> Orchestrator<S> {
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
use crate::WantCreateEventV1;
|
|
||||||
use crate::build_event_log::MemoryBELStorage;
|
use crate::build_event_log::MemoryBELStorage;
|
||||||
use crate::job::JobConfiguration;
|
use crate::job::JobConfiguration;
|
||||||
use crate::mock_job_run::MockJobRun;
|
use crate::mock_job_run::MockJobRun;
|
||||||
use crate::orchestrator::{Orchestrator, OrchestratorConfig};
|
use crate::orchestrator::{Orchestrator, OrchestratorConfig};
|
||||||
use crate::util::current_timestamp;
|
use crate::util::current_timestamp;
|
||||||
|
use crate::want_create_event_v1::Lifetime;
|
||||||
|
use crate::{OriginatingLifetime, WantCreateEventV1};
|
||||||
use uuid::Uuid;
|
use uuid::Uuid;
|
||||||
|
|
||||||
fn build_orchestrator() -> Orchestrator<MemoryBELStorage> {
|
fn build_orchestrator() -> Orchestrator<MemoryBELStorage> {
|
||||||
|
|
@ -477,10 +478,11 @@ mod tests {
|
||||||
Self {
|
Self {
|
||||||
want_id: Uuid::new_v4().to_string(),
|
want_id: Uuid::new_v4().to_string(),
|
||||||
partitions: vec![],
|
partitions: vec![],
|
||||||
data_timestamp: current_timestamp(),
|
lifetime: Some(Lifetime::Originating(OriginatingLifetime {
|
||||||
ttl_seconds: 1000,
|
data_timestamp: current_timestamp(),
|
||||||
sla_seconds: 1000,
|
ttl_seconds: 1000,
|
||||||
source: None,
|
sla_seconds: 1000,
|
||||||
|
})),
|
||||||
comment: Some("test want".to_string()),
|
comment: Some("test want".to_string()),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -1045,7 +1047,8 @@ echo 'Beta succeeded'
|
||||||
mod want_grouping {
|
mod want_grouping {
|
||||||
use super::super::*;
|
use super::super::*;
|
||||||
use crate::build_event_log::MemoryBELStorage;
|
use crate::build_event_log::MemoryBELStorage;
|
||||||
use crate::{PartitionRef, WantDetail};
|
use crate::want_detail::Lifetime;
|
||||||
|
use crate::{OriginatingLifetime, PartitionRef, WantDetail};
|
||||||
|
|
||||||
fn create_job_config(label: &str, pattern: &str) -> JobConfiguration {
|
fn create_job_config(label: &str, pattern: &str) -> JobConfiguration {
|
||||||
JobConfiguration {
|
JobConfiguration {
|
||||||
|
|
@ -1066,13 +1069,17 @@ echo 'Beta succeeded'
|
||||||
})
|
})
|
||||||
.collect(),
|
.collect(),
|
||||||
upstreams: vec![],
|
upstreams: vec![],
|
||||||
data_timestamp: 0,
|
lifetime: Some(Lifetime::Originating(OriginatingLifetime {
|
||||||
ttl_seconds: 0,
|
data_timestamp: 0,
|
||||||
sla_seconds: 0,
|
ttl_seconds: 0,
|
||||||
source: None,
|
sla_seconds: 0,
|
||||||
|
})),
|
||||||
comment: None,
|
comment: None,
|
||||||
status: None,
|
status: None,
|
||||||
last_updated_timestamp: 0,
|
last_updated_timestamp: 0,
|
||||||
|
job_run_ids: vec![],
|
||||||
|
derivative_want_ids: vec![],
|
||||||
|
job_runs: vec![],
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -1,3 +1,4 @@
|
||||||
|
use crate::util::{HasRelatedIds, RelatedIds};
|
||||||
use crate::{PartitionDetail, PartitionRef, PartitionStatus, PartitionStatusCode};
|
use crate::{PartitionDetail, PartitionRef, PartitionStatus, PartitionStatusCode};
|
||||||
use serde::{Deserialize, Serialize};
|
use serde::{Deserialize, Serialize};
|
||||||
use sha2::{Digest, Sha256};
|
use sha2::{Digest, Sha256};
|
||||||
|
|
@ -58,6 +59,8 @@ pub struct UpstreamFailedState {
|
||||||
pub struct TaintedState {
|
pub struct TaintedState {
|
||||||
pub tainted_at: u64,
|
pub tainted_at: u64,
|
||||||
pub taint_ids: Vec<String>,
|
pub taint_ids: Vec<String>,
|
||||||
|
/// Job run that originally built this partition (before it was tainted)
|
||||||
|
pub built_by: String,
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Generic partition struct parameterized by state.
|
/// Generic partition struct parameterized by state.
|
||||||
|
|
@ -250,6 +253,7 @@ impl PartitionWithState<LiveState> {
|
||||||
state: TaintedState {
|
state: TaintedState {
|
||||||
tainted_at: timestamp,
|
tainted_at: timestamp,
|
||||||
taint_ids: vec![taint_id],
|
taint_ids: vec![taint_id],
|
||||||
|
built_by: self.state.built_by,
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -337,9 +341,57 @@ impl Partition {
|
||||||
pub fn is_tainted(&self) -> bool {
|
pub fn is_tainted(&self) -> bool {
|
||||||
matches!(self, Partition::Tainted(_))
|
matches!(self, Partition::Tainted(_))
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ==================== HasRelatedIds trait implementation ====================
|
||||||
|
|
||||||
|
impl HasRelatedIds for Partition {
|
||||||
|
/// Get the IDs of all entities this partition references.
|
||||||
|
/// Note: downstream_partition_uuids and want_ids come from BuildState indexes,
|
||||||
|
/// not from Partition itself.
|
||||||
|
fn related_ids(&self) -> RelatedIds {
|
||||||
|
// Job run ID from the builder (for states that track it)
|
||||||
|
let job_run_ids: Vec<String> = match self {
|
||||||
|
Partition::Building(p) => vec![p.state.job_run_id.clone()],
|
||||||
|
Partition::UpstreamBuilding(p) => vec![p.state.job_run_id.clone()],
|
||||||
|
Partition::UpForRetry(p) => vec![p.state.original_job_run_id.clone()],
|
||||||
|
Partition::Live(p) => vec![p.state.built_by.clone()],
|
||||||
|
Partition::Failed(p) => vec![p.state.failed_by.clone()],
|
||||||
|
Partition::UpstreamFailed(_) => vec![],
|
||||||
|
Partition::Tainted(p) => vec![p.state.built_by.clone()],
|
||||||
|
};
|
||||||
|
|
||||||
|
// Partition refs from missing deps (for UpstreamBuilding state)
|
||||||
|
let partition_refs: Vec<String> = match self {
|
||||||
|
Partition::UpstreamBuilding(p) => p
|
||||||
|
.state
|
||||||
|
.missing_deps
|
||||||
|
.iter()
|
||||||
|
.map(|d| d.r#ref.clone())
|
||||||
|
.collect(),
|
||||||
|
Partition::UpstreamFailed(p) => p
|
||||||
|
.state
|
||||||
|
.failed_upstream_refs
|
||||||
|
.iter()
|
||||||
|
.map(|d| d.r#ref.clone())
|
||||||
|
.collect(),
|
||||||
|
_ => vec![],
|
||||||
|
};
|
||||||
|
|
||||||
|
RelatedIds {
|
||||||
|
partition_refs,
|
||||||
|
partition_uuids: vec![],
|
||||||
|
job_run_ids,
|
||||||
|
want_ids: vec![],
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Partition {
|
||||||
/// Convert to PartitionDetail for API responses and queries.
|
/// Convert to PartitionDetail for API responses and queries.
|
||||||
/// Note: want_ids is now empty - this will be populated by BuildState from the inverted index.
|
/// Note: want_ids and downstream_partition_uuids are empty here and will be
|
||||||
|
/// populated by BuildState from its inverted indexes.
|
||||||
|
/// Upstream lineage is resolved via built_by_job_run_id → job run's read_deps.
|
||||||
pub fn to_detail(&self) -> PartitionDetail {
|
pub fn to_detail(&self) -> PartitionDetail {
|
||||||
match self {
|
match self {
|
||||||
Partition::Building(p) => PartitionDetail {
|
Partition::Building(p) => PartitionDetail {
|
||||||
|
|
@ -353,6 +405,8 @@ impl Partition {
|
||||||
taint_ids: vec![],
|
taint_ids: vec![],
|
||||||
last_updated_timestamp: None,
|
last_updated_timestamp: None,
|
||||||
uuid: p.uuid.to_string(),
|
uuid: p.uuid.to_string(),
|
||||||
|
built_by_job_run_id: None,
|
||||||
|
downstream_partition_uuids: vec![], // Populated by BuildState
|
||||||
},
|
},
|
||||||
Partition::UpstreamBuilding(p) => PartitionDetail {
|
Partition::UpstreamBuilding(p) => PartitionDetail {
|
||||||
r#ref: Some(p.partition_ref.clone()),
|
r#ref: Some(p.partition_ref.clone()),
|
||||||
|
|
@ -365,6 +419,8 @@ impl Partition {
|
||||||
taint_ids: vec![],
|
taint_ids: vec![],
|
||||||
last_updated_timestamp: None,
|
last_updated_timestamp: None,
|
||||||
uuid: p.uuid.to_string(),
|
uuid: p.uuid.to_string(),
|
||||||
|
built_by_job_run_id: None,
|
||||||
|
downstream_partition_uuids: vec![], // Populated by BuildState
|
||||||
},
|
},
|
||||||
Partition::UpForRetry(p) => PartitionDetail {
|
Partition::UpForRetry(p) => PartitionDetail {
|
||||||
r#ref: Some(p.partition_ref.clone()),
|
r#ref: Some(p.partition_ref.clone()),
|
||||||
|
|
@ -377,6 +433,8 @@ impl Partition {
|
||||||
taint_ids: vec![],
|
taint_ids: vec![],
|
||||||
last_updated_timestamp: None,
|
last_updated_timestamp: None,
|
||||||
uuid: p.uuid.to_string(),
|
uuid: p.uuid.to_string(),
|
||||||
|
built_by_job_run_id: None,
|
||||||
|
downstream_partition_uuids: vec![], // Populated by BuildState
|
||||||
},
|
},
|
||||||
Partition::Live(p) => PartitionDetail {
|
Partition::Live(p) => PartitionDetail {
|
||||||
r#ref: Some(p.partition_ref.clone()),
|
r#ref: Some(p.partition_ref.clone()),
|
||||||
|
|
@ -389,6 +447,8 @@ impl Partition {
|
||||||
taint_ids: vec![],
|
taint_ids: vec![],
|
||||||
last_updated_timestamp: Some(p.state.built_at),
|
last_updated_timestamp: Some(p.state.built_at),
|
||||||
uuid: p.uuid.to_string(),
|
uuid: p.uuid.to_string(),
|
||||||
|
built_by_job_run_id: Some(p.state.built_by.clone()),
|
||||||
|
downstream_partition_uuids: vec![], // Populated by BuildState
|
||||||
},
|
},
|
||||||
Partition::Failed(p) => PartitionDetail {
|
Partition::Failed(p) => PartitionDetail {
|
||||||
r#ref: Some(p.partition_ref.clone()),
|
r#ref: Some(p.partition_ref.clone()),
|
||||||
|
|
@ -401,6 +461,8 @@ impl Partition {
|
||||||
taint_ids: vec![],
|
taint_ids: vec![],
|
||||||
last_updated_timestamp: Some(p.state.failed_at),
|
last_updated_timestamp: Some(p.state.failed_at),
|
||||||
uuid: p.uuid.to_string(),
|
uuid: p.uuid.to_string(),
|
||||||
|
built_by_job_run_id: None,
|
||||||
|
downstream_partition_uuids: vec![], // Populated by BuildState
|
||||||
},
|
},
|
||||||
Partition::UpstreamFailed(p) => PartitionDetail {
|
Partition::UpstreamFailed(p) => PartitionDetail {
|
||||||
r#ref: Some(p.partition_ref.clone()),
|
r#ref: Some(p.partition_ref.clone()),
|
||||||
|
|
@ -413,6 +475,8 @@ impl Partition {
|
||||||
taint_ids: vec![],
|
taint_ids: vec![],
|
||||||
last_updated_timestamp: Some(p.state.failed_at),
|
last_updated_timestamp: Some(p.state.failed_at),
|
||||||
uuid: p.uuid.to_string(),
|
uuid: p.uuid.to_string(),
|
||||||
|
built_by_job_run_id: None,
|
||||||
|
downstream_partition_uuids: vec![], // Populated by BuildState
|
||||||
},
|
},
|
||||||
Partition::Tainted(p) => PartitionDetail {
|
Partition::Tainted(p) => PartitionDetail {
|
||||||
r#ref: Some(p.partition_ref.clone()),
|
r#ref: Some(p.partition_ref.clone()),
|
||||||
|
|
@ -421,10 +485,12 @@ impl Partition {
|
||||||
name: "PartitionTainted".to_string(),
|
name: "PartitionTainted".to_string(),
|
||||||
}),
|
}),
|
||||||
want_ids: vec![], // Populated by BuildState
|
want_ids: vec![], // Populated by BuildState
|
||||||
job_run_ids: vec![],
|
job_run_ids: vec![p.state.built_by.clone()],
|
||||||
taint_ids: p.state.taint_ids.clone(),
|
taint_ids: p.state.taint_ids.clone(),
|
||||||
last_updated_timestamp: Some(p.state.tainted_at),
|
last_updated_timestamp: Some(p.state.tainted_at),
|
||||||
uuid: p.uuid.to_string(),
|
uuid: p.uuid.to_string(),
|
||||||
|
built_by_job_run_id: Some(p.state.built_by.clone()),
|
||||||
|
downstream_partition_uuids: vec![], // Populated by BuildState
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -1,5 +1,57 @@
|
||||||
use std::backtrace::Backtrace;
|
use std::backtrace::Backtrace;
|
||||||
use std::time::{SystemTime, UNIX_EPOCH};
|
use std::time::{SystemTime, UNIX_EPOCH};
|
||||||
|
use uuid::Uuid;
|
||||||
|
|
||||||
|
// ============================================================================
|
||||||
|
// Related IDs - for building RelatedEntities index in API responses
|
||||||
|
// ============================================================================
|
||||||
|
|
||||||
|
/// IDs of related entities that an object references.
|
||||||
|
/// Used by the query layer to build the RelatedEntities index for API responses.
|
||||||
|
#[derive(Debug, Clone, Default)]
|
||||||
|
pub struct RelatedIds {
|
||||||
|
/// Partition refs (e.g., from Want.partitions, JobRun.building_partitions)
|
||||||
|
pub partition_refs: Vec<String>,
|
||||||
|
/// Partition UUIDs (e.g., from read/write lineage, consumer index)
|
||||||
|
pub partition_uuids: Vec<Uuid>,
|
||||||
|
/// Job run IDs (e.g., from built_by, consumer jobs)
|
||||||
|
pub job_run_ids: Vec<String>,
|
||||||
|
/// Want IDs (e.g., derivative wants, upstream wants, servicing wants)
|
||||||
|
pub want_ids: Vec<String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl RelatedIds {
|
||||||
|
/// Merge another RelatedIds into this one, deduplicating
|
||||||
|
pub fn merge(&mut self, other: RelatedIds) {
|
||||||
|
for r in other.partition_refs {
|
||||||
|
if !self.partition_refs.contains(&r) {
|
||||||
|
self.partition_refs.push(r);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for u in other.partition_uuids {
|
||||||
|
if !self.partition_uuids.contains(&u) {
|
||||||
|
self.partition_uuids.push(u);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for j in other.job_run_ids {
|
||||||
|
if !self.job_run_ids.contains(&j) {
|
||||||
|
self.job_run_ids.push(j);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for w in other.want_ids {
|
||||||
|
if !self.want_ids.contains(&w) {
|
||||||
|
self.want_ids.push(w);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Trait for entities that can report their related entity IDs.
|
||||||
|
/// Used by the query layer to build RelatedEntities indexes for API responses.
|
||||||
|
/// Implementing types: Want, JobRun, Partition
|
||||||
|
pub trait HasRelatedIds {
|
||||||
|
fn related_ids(&self) -> RelatedIds;
|
||||||
|
}
|
||||||
|
|
||||||
pub fn current_timestamp() -> u64 {
|
pub fn current_timestamp() -> u64 {
|
||||||
let now = SystemTime::now();
|
let now = SystemTime::now();
|
||||||
|
|
@ -89,3 +141,192 @@ impl std::fmt::Display for DatabuildError {
|
||||||
write!(f, "{}", self.msg)
|
write!(f, "{}", self.msg)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ============================================================================
|
||||||
|
// Test Scenarios - reusable BEL event sequences for testing
|
||||||
|
// ============================================================================
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
pub mod test_scenarios {
|
||||||
|
use crate::data_build_event::Event;
|
||||||
|
use crate::want_create_event_v1::Lifetime;
|
||||||
|
use crate::{
|
||||||
|
EphemeralLifetime, JobRunBufferEventV1, JobRunHeartbeatEventV1, JobRunMissingDepsEventV1,
|
||||||
|
JobRunSuccessEventV1, MissingDeps, OriginatingLifetime, PartitionRef, ReadDeps,
|
||||||
|
WantAttributedPartitions, WantCreateEventV1,
|
||||||
|
};
|
||||||
|
|
||||||
|
/// Returns a default Originating lifetime for use in tests.
|
||||||
|
/// WantCreateEventV1 requires a lifetime to be set.
|
||||||
|
pub fn default_originating_lifetime() -> Lifetime {
|
||||||
|
Lifetime::Originating(OriginatingLifetime {
|
||||||
|
data_timestamp: 1000,
|
||||||
|
ttl_seconds: 3600,
|
||||||
|
sla_seconds: 7200,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
/// IDs used in the multihop scenario for easy reference in tests
|
||||||
|
pub struct MultihopIds {
|
||||||
|
pub beta_want_id: String,
|
||||||
|
pub alpha_want_id: String,
|
||||||
|
pub beta_job_1_id: String,
|
||||||
|
pub beta_job_2_id: String,
|
||||||
|
pub alpha_job_id: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Default for MultihopIds {
|
||||||
|
fn default() -> Self {
|
||||||
|
Self {
|
||||||
|
beta_want_id: "beta-want".to_string(),
|
||||||
|
alpha_want_id: "alpha-want".to_string(),
|
||||||
|
beta_job_1_id: "beta-job-1".to_string(),
|
||||||
|
beta_job_2_id: "beta-job-2".to_string(),
|
||||||
|
alpha_job_id: "alpha-job".to_string(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Creates a multihop dependency scenario:
|
||||||
|
/// 1. Want for data/beta is created
|
||||||
|
/// 2. Job beta-job-1 starts, discovers missing dep on data/alpha
|
||||||
|
/// 3. Derivative want for data/alpha is created
|
||||||
|
/// 4. Job alpha-job builds data/alpha successfully
|
||||||
|
/// 5. Job beta-job-2 retries and succeeds
|
||||||
|
///
|
||||||
|
/// This exercises: want creation, job buffering, dep-miss, derivative wants,
|
||||||
|
/// job success with read_deps, and retry logic.
|
||||||
|
pub fn multihop_scenario() -> (Vec<Event>, MultihopIds) {
|
||||||
|
let ids = MultihopIds::default();
|
||||||
|
let mut events = vec![];
|
||||||
|
|
||||||
|
// 1. Create originating want for data/beta (user-requested)
|
||||||
|
events.push(Event::WantCreateV1(WantCreateEventV1 {
|
||||||
|
want_id: ids.beta_want_id.clone(),
|
||||||
|
partitions: vec![PartitionRef {
|
||||||
|
r#ref: "data/beta".to_string(),
|
||||||
|
}],
|
||||||
|
lifetime: Some(Lifetime::Originating(OriginatingLifetime {
|
||||||
|
data_timestamp: 1000,
|
||||||
|
ttl_seconds: 3600,
|
||||||
|
sla_seconds: 7200,
|
||||||
|
})),
|
||||||
|
comment: Some("User requested beta data".to_string()),
|
||||||
|
}));
|
||||||
|
|
||||||
|
// 2. Queue beta job (first attempt)
|
||||||
|
events.push(Event::JobRunBufferV1(JobRunBufferEventV1 {
|
||||||
|
job_run_id: ids.beta_job_1_id.clone(),
|
||||||
|
job_label: "//job_beta".to_string(),
|
||||||
|
want_attributed_partitions: vec![WantAttributedPartitions {
|
||||||
|
want_id: ids.beta_want_id.clone(),
|
||||||
|
partitions: vec![PartitionRef {
|
||||||
|
r#ref: "data/beta".to_string(),
|
||||||
|
}],
|
||||||
|
}],
|
||||||
|
building_partitions: vec![PartitionRef {
|
||||||
|
r#ref: "data/beta".to_string(),
|
||||||
|
}],
|
||||||
|
..Default::default()
|
||||||
|
}));
|
||||||
|
|
||||||
|
// 3. Beta job starts running
|
||||||
|
events.push(Event::JobRunHeartbeatV1(JobRunHeartbeatEventV1 {
|
||||||
|
job_run_id: ids.beta_job_1_id.clone(),
|
||||||
|
..Default::default()
|
||||||
|
}));
|
||||||
|
|
||||||
|
// 4. Beta job reports missing dependency on data/alpha
|
||||||
|
events.push(Event::JobRunMissingDepsV1(JobRunMissingDepsEventV1 {
|
||||||
|
job_run_id: ids.beta_job_1_id.clone(),
|
||||||
|
missing_deps: vec![MissingDeps {
|
||||||
|
impacted: vec![PartitionRef {
|
||||||
|
r#ref: "data/beta".to_string(),
|
||||||
|
}],
|
||||||
|
missing: vec![PartitionRef {
|
||||||
|
r#ref: "data/alpha".to_string(),
|
||||||
|
}],
|
||||||
|
}],
|
||||||
|
..Default::default()
|
||||||
|
}));
|
||||||
|
|
||||||
|
// 5. Create ephemeral want for data/alpha (derivative - created due to dep-miss)
|
||||||
|
events.push(Event::WantCreateV1(WantCreateEventV1 {
|
||||||
|
want_id: ids.alpha_want_id.clone(),
|
||||||
|
partitions: vec![PartitionRef {
|
||||||
|
r#ref: "data/alpha".to_string(),
|
||||||
|
}],
|
||||||
|
lifetime: Some(Lifetime::Ephemeral(EphemeralLifetime {
|
||||||
|
job_run_id: ids.beta_job_1_id.clone(),
|
||||||
|
})),
|
||||||
|
comment: Some("Missing data".to_string()),
|
||||||
|
}));
|
||||||
|
|
||||||
|
// 6. Queue alpha job
|
||||||
|
events.push(Event::JobRunBufferV1(JobRunBufferEventV1 {
|
||||||
|
job_run_id: ids.alpha_job_id.clone(),
|
||||||
|
job_label: "//job_alpha".to_string(),
|
||||||
|
want_attributed_partitions: vec![WantAttributedPartitions {
|
||||||
|
want_id: ids.alpha_want_id.clone(),
|
||||||
|
partitions: vec![PartitionRef {
|
||||||
|
r#ref: "data/alpha".to_string(),
|
||||||
|
}],
|
||||||
|
}],
|
||||||
|
building_partitions: vec![PartitionRef {
|
||||||
|
r#ref: "data/alpha".to_string(),
|
||||||
|
}],
|
||||||
|
..Default::default()
|
||||||
|
}));
|
||||||
|
|
||||||
|
// 7. Alpha job starts running
|
||||||
|
events.push(Event::JobRunHeartbeatV1(JobRunHeartbeatEventV1 {
|
||||||
|
job_run_id: ids.alpha_job_id.clone(),
|
||||||
|
..Default::default()
|
||||||
|
}));
|
||||||
|
|
||||||
|
// 8. Alpha job succeeds (no read deps for leaf node)
|
||||||
|
events.push(Event::JobRunSuccessV1(JobRunSuccessEventV1 {
|
||||||
|
job_run_id: ids.alpha_job_id.clone(),
|
||||||
|
read_deps: vec![],
|
||||||
|
..Default::default()
|
||||||
|
}));
|
||||||
|
|
||||||
|
// 9. Queue beta job again (second attempt - retry)
|
||||||
|
events.push(Event::JobRunBufferV1(JobRunBufferEventV1 {
|
||||||
|
job_run_id: ids.beta_job_2_id.clone(),
|
||||||
|
job_label: "//job_beta".to_string(),
|
||||||
|
want_attributed_partitions: vec![WantAttributedPartitions {
|
||||||
|
want_id: ids.beta_want_id.clone(),
|
||||||
|
partitions: vec![PartitionRef {
|
||||||
|
r#ref: "data/beta".to_string(),
|
||||||
|
}],
|
||||||
|
}],
|
||||||
|
building_partitions: vec![PartitionRef {
|
||||||
|
r#ref: "data/beta".to_string(),
|
||||||
|
}],
|
||||||
|
..Default::default()
|
||||||
|
}));
|
||||||
|
|
||||||
|
// 10. Beta job 2 starts running
|
||||||
|
events.push(Event::JobRunHeartbeatV1(JobRunHeartbeatEventV1 {
|
||||||
|
job_run_id: ids.beta_job_2_id.clone(),
|
||||||
|
..Default::default()
|
||||||
|
}));
|
||||||
|
|
||||||
|
// 11. Beta job 2 succeeds with read_deps showing it read data/alpha
|
||||||
|
events.push(Event::JobRunSuccessV1(JobRunSuccessEventV1 {
|
||||||
|
job_run_id: ids.beta_job_2_id.clone(),
|
||||||
|
read_deps: vec![ReadDeps {
|
||||||
|
impacted: vec![PartitionRef {
|
||||||
|
r#ref: "data/beta".to_string(),
|
||||||
|
}],
|
||||||
|
read: vec![PartitionRef {
|
||||||
|
r#ref: "data/alpha".to_string(),
|
||||||
|
}],
|
||||||
|
}],
|
||||||
|
..Default::default()
|
||||||
|
}));
|
||||||
|
|
||||||
|
(events, ids)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
|
||||||
|
|
@ -1,8 +1,31 @@
|
||||||
use crate::partition_state::FailedPartitionRef;
|
use crate::partition_state::FailedPartitionRef;
|
||||||
use crate::util::current_timestamp;
|
use crate::util::{HasRelatedIds, RelatedIds, current_timestamp};
|
||||||
use crate::{EventSource, PartitionRef, WantCreateEventV1, WantDetail, WantStatusCode};
|
use crate::want_create_event_v1::Lifetime;
|
||||||
|
use crate::want_detail::Lifetime as WantDetailLifetime;
|
||||||
|
use crate::{
|
||||||
|
EphemeralLifetime, EventSource, OriginatingLifetime, PartitionRef, WantCreateEventV1,
|
||||||
|
WantDetail, WantStatusCode,
|
||||||
|
};
|
||||||
use serde::{Deserialize, Serialize};
|
use serde::{Deserialize, Serialize};
|
||||||
use uuid::Uuid;
|
|
||||||
|
/// Want lifetime semantics - determines how freshness/TTL is evaluated
|
||||||
|
#[derive(Debug, Clone)]
|
||||||
|
pub enum WantLifetime {
|
||||||
|
/// User/API-created wants with explicit freshness requirements.
|
||||||
|
/// These drive ongoing rebuilds when partitions get tainted.
|
||||||
|
Originating {
|
||||||
|
data_timestamp: u64,
|
||||||
|
ttl_seconds: u64,
|
||||||
|
sla_seconds: u64,
|
||||||
|
},
|
||||||
|
/// System-created (derivative) wants from dep-miss.
|
||||||
|
/// Delegate freshness decisions to their originating want.
|
||||||
|
/// Complete when partitions become Live, never trigger independent rebuilds.
|
||||||
|
Ephemeral {
|
||||||
|
/// The job run that hit dep-miss and created this derivative want
|
||||||
|
job_run_id: String,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
/// State: Want has just been created, state not yet determined by sensing partition states
|
/// State: Want has just been created, state not yet determined by sensing partition states
|
||||||
#[derive(Debug, Clone)]
|
#[derive(Debug, Clone)]
|
||||||
|
|
@ -57,12 +80,11 @@ pub struct CanceledState {
|
||||||
pub struct WantInfo {
|
pub struct WantInfo {
|
||||||
pub want_id: String,
|
pub want_id: String,
|
||||||
pub partitions: Vec<PartitionRef>,
|
pub partitions: Vec<PartitionRef>,
|
||||||
pub data_timestamp: u64,
|
pub lifetime: WantLifetime,
|
||||||
pub ttl_seconds: u64,
|
|
||||||
pub sla_seconds: u64,
|
|
||||||
pub source: Option<EventSource>,
|
|
||||||
pub comment: Option<String>,
|
pub comment: Option<String>,
|
||||||
pub last_updated_at: u64,
|
pub last_updated_at: u64,
|
||||||
|
/// Job runs that have serviced this want (populated by handle_job_run_buffer)
|
||||||
|
pub job_run_ids: Vec<String>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl Default for WantInfo {
|
impl Default for WantInfo {
|
||||||
|
|
@ -70,12 +92,14 @@ impl Default for WantInfo {
|
||||||
Self {
|
Self {
|
||||||
want_id: uuid::Uuid::new_v4().to_string(),
|
want_id: uuid::Uuid::new_v4().to_string(),
|
||||||
partitions: vec![],
|
partitions: vec![],
|
||||||
data_timestamp: 0,
|
lifetime: WantLifetime::Originating {
|
||||||
ttl_seconds: 0,
|
data_timestamp: 0,
|
||||||
sla_seconds: 0,
|
ttl_seconds: 0,
|
||||||
source: None,
|
sla_seconds: 0,
|
||||||
|
},
|
||||||
comment: None,
|
comment: None,
|
||||||
last_updated_at: 0,
|
last_updated_at: 0,
|
||||||
|
job_run_ids: vec![],
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -180,16 +204,26 @@ impl WantWithState<CanceledState> {
|
||||||
// From impl for creating want from event - creates in New state for sensing
|
// From impl for creating want from event - creates in New state for sensing
|
||||||
impl From<WantCreateEventV1> for WantWithState<NewState> {
|
impl From<WantCreateEventV1> for WantWithState<NewState> {
|
||||||
fn from(event: WantCreateEventV1) -> Self {
|
fn from(event: WantCreateEventV1) -> Self {
|
||||||
|
let lifetime = match event.lifetime {
|
||||||
|
Some(Lifetime::Originating(orig)) => WantLifetime::Originating {
|
||||||
|
data_timestamp: orig.data_timestamp,
|
||||||
|
ttl_seconds: orig.ttl_seconds,
|
||||||
|
sla_seconds: orig.sla_seconds,
|
||||||
|
},
|
||||||
|
Some(Lifetime::Ephemeral(eph)) => WantLifetime::Ephemeral {
|
||||||
|
job_run_id: eph.job_run_id,
|
||||||
|
},
|
||||||
|
None => panic!("Unexpectedly empty want lifetime"),
|
||||||
|
};
|
||||||
|
|
||||||
WantWithState {
|
WantWithState {
|
||||||
want: WantInfo {
|
want: WantInfo {
|
||||||
want_id: event.want_id,
|
want_id: event.want_id,
|
||||||
partitions: event.partitions,
|
partitions: event.partitions,
|
||||||
data_timestamp: event.data_timestamp,
|
lifetime,
|
||||||
ttl_seconds: event.ttl_seconds,
|
|
||||||
sla_seconds: event.sla_seconds,
|
|
||||||
source: event.source,
|
|
||||||
comment: event.comment,
|
comment: event.comment,
|
||||||
last_updated_at: current_timestamp(),
|
last_updated_at: current_timestamp(),
|
||||||
|
job_run_ids: vec![],
|
||||||
},
|
},
|
||||||
state: NewState {},
|
state: NewState {},
|
||||||
}
|
}
|
||||||
|
|
@ -464,33 +498,134 @@ impl WantWithState<UpstreamBuildingState> {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ==================== HasRelatedIds trait implementation ====================
|
||||||
|
|
||||||
|
impl HasRelatedIds for Want {
|
||||||
|
/// Get the IDs of all entities this want references.
|
||||||
|
/// Note: job_run_ids come from inverted indexes in BuildState, not from Want itself.
|
||||||
|
fn related_ids(&self) -> RelatedIds {
|
||||||
|
let partition_refs = self
|
||||||
|
.want()
|
||||||
|
.partitions
|
||||||
|
.iter()
|
||||||
|
.map(|p| p.r#ref.clone())
|
||||||
|
.collect();
|
||||||
|
|
||||||
|
// Collect want IDs from state-specific relationships
|
||||||
|
let want_ids = match self {
|
||||||
|
Want::UpstreamBuilding(w) => w.state.upstream_want_ids.clone(),
|
||||||
|
Want::UpstreamFailed(w) => w.state.failed_wants.clone(),
|
||||||
|
_ => vec![],
|
||||||
|
};
|
||||||
|
|
||||||
|
RelatedIds {
|
||||||
|
partition_refs,
|
||||||
|
partition_uuids: vec![],
|
||||||
|
job_run_ids: vec![],
|
||||||
|
want_ids,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// Helper methods on the Want enum
|
// Helper methods on the Want enum
|
||||||
impl Want {
|
impl Want {
|
||||||
/// Create a new want in the Idle state
|
/// Create a new originating want in the Idle state
|
||||||
pub fn new(
|
pub fn new_originating(
|
||||||
want_id: String,
|
want_id: String,
|
||||||
partitions: Vec<PartitionRef>,
|
partitions: Vec<PartitionRef>,
|
||||||
data_timestamp: u64,
|
data_timestamp: u64,
|
||||||
ttl_seconds: u64,
|
ttl_seconds: u64,
|
||||||
sla_seconds: u64,
|
sla_seconds: u64,
|
||||||
source: Option<EventSource>,
|
|
||||||
comment: Option<String>,
|
comment: Option<String>,
|
||||||
) -> Self {
|
) -> Self {
|
||||||
Want::Idle(WantWithState {
|
Want::Idle(WantWithState {
|
||||||
want: WantInfo {
|
want: WantInfo {
|
||||||
want_id,
|
want_id,
|
||||||
partitions,
|
partitions,
|
||||||
data_timestamp,
|
lifetime: WantLifetime::Originating {
|
||||||
ttl_seconds,
|
data_timestamp,
|
||||||
sla_seconds,
|
ttl_seconds,
|
||||||
source,
|
sla_seconds,
|
||||||
|
},
|
||||||
comment,
|
comment,
|
||||||
last_updated_at: current_timestamp(),
|
last_updated_at: current_timestamp(),
|
||||||
|
job_run_ids: vec![],
|
||||||
},
|
},
|
||||||
state: IdleState {},
|
state: IdleState {},
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Create a new ephemeral want in the Idle state (derivative from dep-miss)
|
||||||
|
pub fn new_ephemeral(
|
||||||
|
want_id: String,
|
||||||
|
partitions: Vec<PartitionRef>,
|
||||||
|
job_run_id: String,
|
||||||
|
comment: Option<String>,
|
||||||
|
) -> Self {
|
||||||
|
Want::Idle(WantWithState {
|
||||||
|
want: WantInfo {
|
||||||
|
want_id,
|
||||||
|
partitions,
|
||||||
|
lifetime: WantLifetime::Ephemeral { job_run_id },
|
||||||
|
comment,
|
||||||
|
last_updated_at: current_timestamp(),
|
||||||
|
job_run_ids: vec![],
|
||||||
|
},
|
||||||
|
state: IdleState {},
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Get the lifetime of this want
|
||||||
|
pub fn lifetime(&self) -> &WantLifetime {
|
||||||
|
&self.want().lifetime
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Add a job run ID to this want's list of servicing job runs
|
||||||
|
pub fn add_job_run_id(&mut self, job_run_id: &str) {
|
||||||
|
match self {
|
||||||
|
Want::New(w) => {
|
||||||
|
if !w.want.job_run_ids.contains(&job_run_id.to_string()) {
|
||||||
|
w.want.job_run_ids.push(job_run_id.to_string());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Want::Idle(w) => {
|
||||||
|
if !w.want.job_run_ids.contains(&job_run_id.to_string()) {
|
||||||
|
w.want.job_run_ids.push(job_run_id.to_string());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Want::Building(w) => {
|
||||||
|
if !w.want.job_run_ids.contains(&job_run_id.to_string()) {
|
||||||
|
w.want.job_run_ids.push(job_run_id.to_string());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Want::UpstreamBuilding(w) => {
|
||||||
|
if !w.want.job_run_ids.contains(&job_run_id.to_string()) {
|
||||||
|
w.want.job_run_ids.push(job_run_id.to_string());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Want::Successful(w) => {
|
||||||
|
if !w.want.job_run_ids.contains(&job_run_id.to_string()) {
|
||||||
|
w.want.job_run_ids.push(job_run_id.to_string());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Want::Failed(w) => {
|
||||||
|
if !w.want.job_run_ids.contains(&job_run_id.to_string()) {
|
||||||
|
w.want.job_run_ids.push(job_run_id.to_string());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Want::UpstreamFailed(w) => {
|
||||||
|
if !w.want.job_run_ids.contains(&job_run_id.to_string()) {
|
||||||
|
w.want.job_run_ids.push(job_run_id.to_string());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Want::Canceled(w) => {
|
||||||
|
if !w.want.job_run_ids.contains(&job_run_id.to_string()) {
|
||||||
|
w.want.job_run_ids.push(job_run_id.to_string());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// Check if want is schedulable (Idle or UpstreamBuilding with satisfied upstreams)
|
/// Check if want is schedulable (Idle or UpstreamBuilding with satisfied upstreams)
|
||||||
pub fn is_schedulable(&self) -> bool {
|
pub fn is_schedulable(&self) -> bool {
|
||||||
match self {
|
match self {
|
||||||
|
|
@ -522,16 +657,32 @@ impl Want {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Convert to WantDetail for API responses and queries
|
/// Convert to WantDetail for API responses and queries.
|
||||||
|
/// job_run_ids are returned from the Want itself.
|
||||||
|
/// derivative_want_ids are computed by traversing job runs (done by BuildState).
|
||||||
pub fn to_detail(&self) -> WantDetail {
|
pub fn to_detail(&self) -> WantDetail {
|
||||||
|
let lifetime = match &self.want().lifetime {
|
||||||
|
WantLifetime::Originating {
|
||||||
|
data_timestamp,
|
||||||
|
ttl_seconds,
|
||||||
|
sla_seconds,
|
||||||
|
} => Some(WantDetailLifetime::Originating(OriginatingLifetime {
|
||||||
|
data_timestamp: *data_timestamp,
|
||||||
|
ttl_seconds: *ttl_seconds,
|
||||||
|
sla_seconds: *sla_seconds,
|
||||||
|
})),
|
||||||
|
WantLifetime::Ephemeral { job_run_id } => {
|
||||||
|
Some(WantDetailLifetime::Ephemeral(EphemeralLifetime {
|
||||||
|
job_run_id: job_run_id.clone(),
|
||||||
|
}))
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
WantDetail {
|
WantDetail {
|
||||||
want_id: self.want().want_id.clone(),
|
want_id: self.want().want_id.clone(),
|
||||||
partitions: self.want().partitions.clone(),
|
partitions: self.want().partitions.clone(),
|
||||||
upstreams: vec![], // Upstreams are tracked via want relationships, not stored here
|
upstreams: vec![], // Upstreams are tracked via want relationships, not stored here
|
||||||
data_timestamp: self.want().data_timestamp,
|
lifetime,
|
||||||
ttl_seconds: self.want().ttl_seconds,
|
|
||||||
sla_seconds: self.want().sla_seconds,
|
|
||||||
source: self.want().source.clone(),
|
|
||||||
comment: self.want().comment.clone(),
|
comment: self.want().comment.clone(),
|
||||||
last_updated_timestamp: self.want().last_updated_at,
|
last_updated_timestamp: self.want().last_updated_at,
|
||||||
status: match self {
|
status: match self {
|
||||||
|
|
@ -544,6 +695,9 @@ impl Want {
|
||||||
Want::UpstreamFailed(_) => Some(WantStatusCode::WantUpstreamFailed.into()),
|
Want::UpstreamFailed(_) => Some(WantStatusCode::WantUpstreamFailed.into()),
|
||||||
Want::Canceled(_) => Some(WantStatusCode::WantCanceled.into()),
|
Want::Canceled(_) => Some(WantStatusCode::WantCanceled.into()),
|
||||||
},
|
},
|
||||||
|
job_run_ids: self.want().job_run_ids.clone(),
|
||||||
|
derivative_want_ids: vec![], // Computed by BuildState via job traversal
|
||||||
|
job_runs: vec![], // Populated by BuildState.get_want()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -3,7 +3,7 @@
|
||||||
use askama::Template;
|
use askama::Template;
|
||||||
|
|
||||||
use crate::{
|
use crate::{
|
||||||
JobRunDetail, JobRunStatus, PartitionDetail, PartitionRef, PartitionStatus,
|
JobRunDetail, JobRunStatus, PartitionDetail, PartitionRef, PartitionStatus, ReadDeps,
|
||||||
WantAttributedPartitions, WantDetail, WantStatus,
|
WantAttributedPartitions, WantDetail, WantStatus,
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
@ -73,39 +73,120 @@ impl From<&JobRunStatus> for JobRunStatusView {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
pub struct WantDetailView {
|
/// Simple view for derivative wants in the want detail page
|
||||||
|
pub struct DerivativeWantView {
|
||||||
pub want_id: String,
|
pub want_id: String,
|
||||||
pub partitions: Vec<PartitionRefView>,
|
pub partitions: Vec<PartitionRefView>,
|
||||||
pub upstreams: Vec<PartitionRefView>,
|
|
||||||
pub data_timestamp: u64,
|
|
||||||
pub ttl_seconds: u64,
|
|
||||||
pub sla_seconds: u64,
|
|
||||||
pub comment: Option<String>,
|
|
||||||
pub comment_display: String,
|
|
||||||
pub status: Option<WantStatusView>,
|
pub status: Option<WantStatusView>,
|
||||||
pub last_updated_timestamp: u64,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
impl From<&WantDetail> for WantDetailView {
|
impl From<&WantDetail> for DerivativeWantView {
|
||||||
fn from(w: &WantDetail) -> Self {
|
fn from(w: &WantDetail) -> Self {
|
||||||
Self {
|
Self {
|
||||||
want_id: w.want_id.clone(),
|
want_id: w.want_id.clone(),
|
||||||
partitions: w.partitions.iter().map(PartitionRefView::from).collect(),
|
partitions: w.partitions.iter().map(PartitionRefView::from).collect(),
|
||||||
upstreams: w.upstreams.iter().map(PartitionRefView::from).collect(),
|
|
||||||
data_timestamp: w.data_timestamp,
|
|
||||||
ttl_seconds: w.ttl_seconds,
|
|
||||||
sla_seconds: w.sla_seconds,
|
|
||||||
comment: w.comment.clone(),
|
|
||||||
comment_display: w.comment.as_deref().unwrap_or("-").to_string(),
|
|
||||||
status: w.status.as_ref().map(WantStatusView::from),
|
status: w.status.as_ref().map(WantStatusView::from),
|
||||||
last_updated_timestamp: w.last_updated_timestamp,
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Enum representing the want lifetime type for templates
|
||||||
|
pub enum WantLifetimeView {
|
||||||
|
Originating {
|
||||||
|
data_timestamp: u64,
|
||||||
|
ttl_seconds: u64,
|
||||||
|
sla_seconds: u64,
|
||||||
|
},
|
||||||
|
Ephemeral {
|
||||||
|
job_run_id: String,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
pub struct WantDetailView {
|
||||||
|
pub want_id: String,
|
||||||
|
pub partitions: Vec<PartitionRefView>,
|
||||||
|
pub upstreams: Vec<PartitionRefView>,
|
||||||
|
pub lifetime: Option<WantLifetimeView>,
|
||||||
|
/// Convenience accessor for originating wants - returns data_timestamp or 0
|
||||||
|
pub data_timestamp: u64,
|
||||||
|
/// Convenience accessor for originating wants - returns ttl_seconds or 0
|
||||||
|
pub ttl_seconds: u64,
|
||||||
|
/// Convenience accessor for originating wants - returns sla_seconds or 0
|
||||||
|
pub sla_seconds: u64,
|
||||||
|
/// True if this is an ephemeral (derivative) want
|
||||||
|
pub is_ephemeral: bool,
|
||||||
|
/// Job run that created this ephemeral want (if ephemeral)
|
||||||
|
pub source_job_run_id: Option<String>,
|
||||||
|
pub comment: Option<String>,
|
||||||
|
pub comment_display: String,
|
||||||
|
pub status: Option<WantStatusView>,
|
||||||
|
pub last_updated_timestamp: u64,
|
||||||
|
// Lineage fields
|
||||||
|
pub job_run_ids: Vec<String>,
|
||||||
|
pub derivative_want_ids: Vec<String>,
|
||||||
|
pub job_runs: Vec<JobRunDetailView>,
|
||||||
|
pub derivative_wants: Vec<DerivativeWantView>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl WantDetailView {
|
||||||
|
/// Create a WantDetailView with derivative wants populated.
|
||||||
|
/// Use this for the detail page where derivative wants need to be shown.
|
||||||
|
pub fn new(w: &WantDetail, derivative_wants: Vec<DerivativeWantView>) -> Self {
|
||||||
|
use crate::want_detail::Lifetime;
|
||||||
|
|
||||||
|
let (lifetime, data_timestamp, ttl_seconds, sla_seconds, is_ephemeral, source_job_run_id) =
|
||||||
|
match &w.lifetime {
|
||||||
|
Some(Lifetime::Originating(orig)) => (
|
||||||
|
Some(WantLifetimeView::Originating {
|
||||||
|
data_timestamp: orig.data_timestamp,
|
||||||
|
ttl_seconds: orig.ttl_seconds,
|
||||||
|
sla_seconds: orig.sla_seconds,
|
||||||
|
}),
|
||||||
|
orig.data_timestamp,
|
||||||
|
orig.ttl_seconds,
|
||||||
|
orig.sla_seconds,
|
||||||
|
false,
|
||||||
|
None,
|
||||||
|
),
|
||||||
|
Some(Lifetime::Ephemeral(eph)) => (
|
||||||
|
Some(WantLifetimeView::Ephemeral {
|
||||||
|
job_run_id: eph.job_run_id.clone(),
|
||||||
|
}),
|
||||||
|
0,
|
||||||
|
0,
|
||||||
|
0,
|
||||||
|
true,
|
||||||
|
Some(eph.job_run_id.clone()),
|
||||||
|
),
|
||||||
|
None => (None, 0, 0, 0, false, None),
|
||||||
|
};
|
||||||
|
|
||||||
|
Self {
|
||||||
|
want_id: w.want_id.clone(),
|
||||||
|
partitions: w.partitions.iter().map(PartitionRefView::from).collect(),
|
||||||
|
upstreams: w.upstreams.iter().map(PartitionRefView::from).collect(),
|
||||||
|
lifetime,
|
||||||
|
data_timestamp,
|
||||||
|
ttl_seconds,
|
||||||
|
sla_seconds,
|
||||||
|
is_ephemeral,
|
||||||
|
source_job_run_id,
|
||||||
|
comment: w.comment.clone(),
|
||||||
|
comment_display: w.comment.as_deref().unwrap_or("-").to_string(),
|
||||||
|
status: w.status.as_ref().map(WantStatusView::from),
|
||||||
|
last_updated_timestamp: w.last_updated_timestamp,
|
||||||
|
job_run_ids: w.job_run_ids.clone(),
|
||||||
|
derivative_want_ids: w.derivative_want_ids.clone(),
|
||||||
|
job_runs: w.job_runs.iter().map(JobRunDetailView::from).collect(),
|
||||||
|
derivative_wants,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// For list pages where derivative wants aren't needed
|
||||||
impl From<WantDetail> for WantDetailView {
|
impl From<WantDetail> for WantDetailView {
|
||||||
fn from(w: WantDetail) -> Self {
|
fn from(w: WantDetail) -> Self {
|
||||||
Self::from(&w)
|
Self::new(&w, vec![])
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -119,6 +200,9 @@ pub struct PartitionDetailView {
|
||||||
pub want_ids: Vec<String>,
|
pub want_ids: Vec<String>,
|
||||||
pub taint_ids: Vec<String>,
|
pub taint_ids: Vec<String>,
|
||||||
pub uuid: String,
|
pub uuid: String,
|
||||||
|
// Lineage fields
|
||||||
|
pub built_by_job_run_id: Option<String>,
|
||||||
|
pub downstream_partition_uuids: Vec<String>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl From<&PartitionDetail> for PartitionDetailView {
|
impl From<&PartitionDetail> for PartitionDetailView {
|
||||||
|
|
@ -141,6 +225,8 @@ impl From<&PartitionDetail> for PartitionDetailView {
|
||||||
want_ids: p.want_ids.clone(),
|
want_ids: p.want_ids.clone(),
|
||||||
taint_ids: p.taint_ids.clone(),
|
taint_ids: p.taint_ids.clone(),
|
||||||
uuid: p.uuid.clone(),
|
uuid: p.uuid.clone(),
|
||||||
|
built_by_job_run_id: p.built_by_job_run_id.clone(),
|
||||||
|
downstream_partition_uuids: p.downstream_partition_uuids.clone(),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -165,20 +251,75 @@ impl From<&WantAttributedPartitions> for WantAttributedPartitionsView {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// View for read dependency entries (impacted → read relationships)
|
||||||
|
pub struct ReadDepsView {
|
||||||
|
pub impacted: Vec<PartitionRefView>,
|
||||||
|
pub read: Vec<PartitionRefView>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl From<&ReadDeps> for ReadDepsView {
|
||||||
|
fn from(rd: &ReadDeps) -> Self {
|
||||||
|
Self {
|
||||||
|
impacted: rd.impacted.iter().map(PartitionRefView::from).collect(),
|
||||||
|
read: rd.read.iter().map(PartitionRefView::from).collect(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// View for partition ref with its resolved UUID (for lineage display)
|
||||||
|
pub struct PartitionRefWithUuidView {
|
||||||
|
pub partition_ref: String,
|
||||||
|
pub partition_ref_encoded: String,
|
||||||
|
pub uuid: String,
|
||||||
|
}
|
||||||
|
|
||||||
pub struct JobRunDetailView {
|
pub struct JobRunDetailView {
|
||||||
pub id: String,
|
pub id: String,
|
||||||
|
pub job_label: String,
|
||||||
pub status: Option<JobRunStatusView>,
|
pub status: Option<JobRunStatusView>,
|
||||||
pub last_heartbeat_at: Option<u64>,
|
pub last_heartbeat_at: Option<u64>,
|
||||||
|
pub queued_at: Option<u64>,
|
||||||
|
pub started_at: Option<u64>,
|
||||||
pub building_partitions: Vec<PartitionRefView>,
|
pub building_partitions: Vec<PartitionRefView>,
|
||||||
pub servicing_wants: Vec<WantAttributedPartitionsView>,
|
pub servicing_wants: Vec<WantAttributedPartitionsView>,
|
||||||
|
// Lineage fields (populated for Succeeded/DepMiss states)
|
||||||
|
pub read_deps: Vec<ReadDepsView>,
|
||||||
|
pub read_partitions: Vec<PartitionRefWithUuidView>,
|
||||||
|
pub wrote_partitions: Vec<PartitionRefWithUuidView>,
|
||||||
|
pub derivative_want_ids: Vec<String>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl From<&JobRunDetail> for JobRunDetailView {
|
impl From<&JobRunDetail> for JobRunDetailView {
|
||||||
fn from(jr: &JobRunDetail) -> Self {
|
fn from(jr: &JobRunDetail) -> Self {
|
||||||
|
// Build read_partitions from read_partition_uuids map
|
||||||
|
let read_partitions: Vec<PartitionRefWithUuidView> = jr
|
||||||
|
.read_partition_uuids
|
||||||
|
.iter()
|
||||||
|
.map(|(partition_ref, uuid)| PartitionRefWithUuidView {
|
||||||
|
partition_ref: partition_ref.clone(),
|
||||||
|
partition_ref_encoded: urlencoding::encode(partition_ref).into_owned(),
|
||||||
|
uuid: uuid.clone(),
|
||||||
|
})
|
||||||
|
.collect();
|
||||||
|
|
||||||
|
// Build wrote_partitions from wrote_partition_uuids map
|
||||||
|
let wrote_partitions: Vec<PartitionRefWithUuidView> = jr
|
||||||
|
.wrote_partition_uuids
|
||||||
|
.iter()
|
||||||
|
.map(|(partition_ref, uuid)| PartitionRefWithUuidView {
|
||||||
|
partition_ref: partition_ref.clone(),
|
||||||
|
partition_ref_encoded: urlencoding::encode(partition_ref).into_owned(),
|
||||||
|
uuid: uuid.clone(),
|
||||||
|
})
|
||||||
|
.collect();
|
||||||
|
|
||||||
Self {
|
Self {
|
||||||
id: jr.id.clone(),
|
id: jr.id.clone(),
|
||||||
|
job_label: jr.job_label.clone(),
|
||||||
status: jr.status.as_ref().map(JobRunStatusView::from),
|
status: jr.status.as_ref().map(JobRunStatusView::from),
|
||||||
last_heartbeat_at: jr.last_heartbeat_at,
|
last_heartbeat_at: jr.last_heartbeat_at,
|
||||||
|
queued_at: jr.queued_at,
|
||||||
|
started_at: jr.started_at,
|
||||||
building_partitions: jr
|
building_partitions: jr
|
||||||
.building_partitions
|
.building_partitions
|
||||||
.iter()
|
.iter()
|
||||||
|
|
@ -189,6 +330,10 @@ impl From<&JobRunDetail> for JobRunDetailView {
|
||||||
.iter()
|
.iter()
|
||||||
.map(WantAttributedPartitionsView::from)
|
.map(WantAttributedPartitionsView::from)
|
||||||
.collect(),
|
.collect(),
|
||||||
|
read_deps: jr.read_deps.iter().map(ReadDepsView::from).collect(),
|
||||||
|
read_partitions,
|
||||||
|
wrote_partitions,
|
||||||
|
derivative_want_ids: jr.derivative_want_ids.clone(),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -361,3 +506,100 @@ pub struct JobRunDetailPage {
|
||||||
pub struct WantCreatePage {
|
pub struct WantCreatePage {
|
||||||
pub base: BaseContext,
|
pub base: BaseContext,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// =============================================================================
|
||||||
|
// Tests
|
||||||
|
// =============================================================================
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
use crate::build_state::BuildState;
|
||||||
|
use crate::util::test_scenarios::multihop_scenario;
|
||||||
|
use askama::Template;
|
||||||
|
|
||||||
|
/// Helper to replay events into a fresh BuildState
|
||||||
|
fn build_state_from_events(events: &[crate::data_build_event::Event]) -> BuildState {
|
||||||
|
let mut state = BuildState::default();
|
||||||
|
for event in events {
|
||||||
|
state.handle_event(event);
|
||||||
|
}
|
||||||
|
state
|
||||||
|
}
|
||||||
|
|
||||||
|
mod want_detail_page {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
/// Tests that the want detail page shows the job runs that serviced this want.
|
||||||
|
/// This is the "Fulfillment - Job Runs" section in the UI.
|
||||||
|
///
|
||||||
|
/// Given: The multihop scenario completes (beta job dep-misses, alpha built, beta retries)
|
||||||
|
/// When: We render the beta want detail page
|
||||||
|
/// Then: It should show both beta job runs (beta-job-1 and beta-job-2)
|
||||||
|
#[test]
|
||||||
|
fn test_shows_servicing_job_runs() {
|
||||||
|
let (events, ids) = multihop_scenario();
|
||||||
|
let state = build_state_from_events(&events);
|
||||||
|
|
||||||
|
// Get beta want and render its detail page
|
||||||
|
let want_detail = state
|
||||||
|
.get_want(&ids.beta_want_id)
|
||||||
|
.expect("beta want should exist");
|
||||||
|
let template = WantDetailPage {
|
||||||
|
base: BaseContext::default(),
|
||||||
|
want: WantDetailView::from(want_detail),
|
||||||
|
};
|
||||||
|
let html = template.render().expect("template should render");
|
||||||
|
|
||||||
|
// Verify the Fulfillment section exists and contains both job run IDs
|
||||||
|
assert!(
|
||||||
|
html.contains(&ids.beta_job_1_id),
|
||||||
|
"Should show beta-job-1 in fulfillment section. HTML:\n{}",
|
||||||
|
html
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
html.contains(&ids.beta_job_2_id),
|
||||||
|
"Should show beta-job-2 in fulfillment section. HTML:\n{}",
|
||||||
|
html
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Tests that the want detail page shows derivative wants spawned by dep-miss.
|
||||||
|
/// This is the "Fulfillment - Derivative Wants" section in the UI.
|
||||||
|
///
|
||||||
|
/// Given: The multihop scenario completes (beta job dep-misses, spawning alpha want)
|
||||||
|
/// When: We render the beta want detail page
|
||||||
|
/// Then: It should show the alpha want as a derivative
|
||||||
|
#[test]
|
||||||
|
fn test_shows_derivative_wants() {
|
||||||
|
let (events, ids) = multihop_scenario();
|
||||||
|
let state = build_state_from_events(&events);
|
||||||
|
|
||||||
|
// Get beta want and render its detail page
|
||||||
|
let want_detail = state
|
||||||
|
.get_want(&ids.beta_want_id)
|
||||||
|
.expect("beta want should exist");
|
||||||
|
|
||||||
|
// Fetch derivative wants (like the http_server does)
|
||||||
|
let derivative_wants: Vec<_> = want_detail
|
||||||
|
.derivative_want_ids
|
||||||
|
.iter()
|
||||||
|
.filter_map(|id| state.get_want(id))
|
||||||
|
.map(|w| DerivativeWantView::from(&w))
|
||||||
|
.collect();
|
||||||
|
|
||||||
|
let template = WantDetailPage {
|
||||||
|
base: BaseContext::default(),
|
||||||
|
want: WantDetailView::new(&want_detail, derivative_wants),
|
||||||
|
};
|
||||||
|
let html = template.render().expect("template should render");
|
||||||
|
|
||||||
|
// Verify the Fulfillment section exists and contains the derivative want
|
||||||
|
assert!(
|
||||||
|
html.contains(&ids.alpha_want_id),
|
||||||
|
"Should show alpha-want as derivative want. HTML:\n{}",
|
||||||
|
html
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
|
||||||
|
|
@ -54,4 +54,43 @@
|
||||||
</div>
|
</div>
|
||||||
{% endif %}
|
{% endif %}
|
||||||
|
|
||||||
|
{% if !job_run.read_partitions.is_empty() %}
|
||||||
|
<div class="detail-section">
|
||||||
|
<h2>Read Partitions ({{ job_run.read_partitions.len() }})</h2>
|
||||||
|
<ul class="partition-list">
|
||||||
|
{% for p in job_run.read_partitions %}
|
||||||
|
<li>
|
||||||
|
<a href="/partitions/{{ p.partition_ref_encoded }}" class="partition-ref">{{ p.partition_ref }}</a>
|
||||||
|
<span style="color:var(--color-text-muted);font-size:.75rem;font-family:monospace;margin-left:.5rem">uuid: {{ p.uuid }}</span>
|
||||||
|
</li>
|
||||||
|
{% endfor %}
|
||||||
|
</ul>
|
||||||
|
</div>
|
||||||
|
{% endif %}
|
||||||
|
|
||||||
|
{% if !job_run.wrote_partitions.is_empty() %}
|
||||||
|
<div class="detail-section">
|
||||||
|
<h2>Wrote Partitions ({{ job_run.wrote_partitions.len() }})</h2>
|
||||||
|
<ul class="partition-list">
|
||||||
|
{% for p in job_run.wrote_partitions %}
|
||||||
|
<li>
|
||||||
|
<a href="/partitions/{{ p.partition_ref_encoded }}" class="partition-ref">{{ p.partition_ref }}</a>
|
||||||
|
<span style="color:var(--color-text-muted);font-size:.75rem;font-family:monospace;margin-left:.5rem">uuid: {{ p.uuid }}</span>
|
||||||
|
</li>
|
||||||
|
{% endfor %}
|
||||||
|
</ul>
|
||||||
|
</div>
|
||||||
|
{% endif %}
|
||||||
|
|
||||||
|
{% if !job_run.derivative_want_ids.is_empty() %}
|
||||||
|
<div class="detail-section">
|
||||||
|
<h2>Derivative Wants ({{ job_run.derivative_want_ids.len() }})</h2>
|
||||||
|
<ul class="partition-list">
|
||||||
|
{% for id in job_run.derivative_want_ids %}
|
||||||
|
<li><a href="/wants/{{ id }}">{{ id }}</a></li>
|
||||||
|
{% endfor %}
|
||||||
|
</ul>
|
||||||
|
</div>
|
||||||
|
{% endif %}
|
||||||
|
|
||||||
{% call base::footer() %}
|
{% call base::footer() %}
|
||||||
|
|
|
||||||
|
|
@ -32,9 +32,34 @@
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
|
{% match partition.built_by_job_run_id %}
|
||||||
|
{% when Some with (job_run_id) %}
|
||||||
|
<div class="detail-section">
|
||||||
|
<h2>Lineage - Built By</h2>
|
||||||
|
<p>
|
||||||
|
<a href="/job_runs/{{ job_run_id }}">{{ job_run_id }}</a>
|
||||||
|
<span style="color:var(--color-text-muted);font-size:.75rem;margin-left:.5rem">(view job run for input partitions)</span>
|
||||||
|
</p>
|
||||||
|
</div>
|
||||||
|
{% when None %}
|
||||||
|
{% endmatch %}
|
||||||
|
|
||||||
|
{% if !partition.downstream_partition_uuids.is_empty() %}
|
||||||
|
<div class="detail-section">
|
||||||
|
<h2>Lineage - Downstream Consumers ({{ partition.downstream_partition_uuids.len() }})</h2>
|
||||||
|
<ul class="partition-list">
|
||||||
|
{% for uuid in partition.downstream_partition_uuids %}
|
||||||
|
<li>
|
||||||
|
<span style="font-family:monospace;font-size:.8125rem">{{ uuid }}</span>
|
||||||
|
</li>
|
||||||
|
{% endfor %}
|
||||||
|
</ul>
|
||||||
|
</div>
|
||||||
|
{% endif %}
|
||||||
|
|
||||||
{% if !partition.job_run_ids.is_empty() %}
|
{% if !partition.job_run_ids.is_empty() %}
|
||||||
<div class="detail-section">
|
<div class="detail-section">
|
||||||
<h2>Job Runs ({{ partition.job_run_ids.len() }})</h2>
|
<h2>Related Job Runs ({{ partition.job_run_ids.len() }})</h2>
|
||||||
<ul class="partition-list">
|
<ul class="partition-list">
|
||||||
{% for id in partition.job_run_ids %}
|
{% for id in partition.job_run_ids %}
|
||||||
<li><a href="/job_runs/{{ id }}">{{ id }}</a></li>
|
<li><a href="/job_runs/{{ id }}">{{ id }}</a></li>
|
||||||
|
|
|
||||||
|
|
@ -65,4 +65,56 @@
|
||||||
</div>
|
</div>
|
||||||
{% endif %}
|
{% endif %}
|
||||||
|
|
||||||
|
{% if !want.job_runs.is_empty() %}
|
||||||
|
<div class="detail-section">
|
||||||
|
<h2>Fulfillment - Job Runs ({{ want.job_runs.len() }})</h2>
|
||||||
|
<table class="data-table">
|
||||||
|
<thead>
|
||||||
|
<tr>
|
||||||
|
<th>ID</th>
|
||||||
|
<th>Job Label</th>
|
||||||
|
<th>Started</th>
|
||||||
|
<th>Duration</th>
|
||||||
|
<th>Status</th>
|
||||||
|
</tr>
|
||||||
|
</thead>
|
||||||
|
<tbody>
|
||||||
|
{% for jr in want.job_runs %}
|
||||||
|
<tr>
|
||||||
|
<td><a href="/job_runs/{{ jr.id }}">{{ jr.id }}</a></td>
|
||||||
|
<td>{{ jr.job_label }}</td>
|
||||||
|
<td>{% match jr.started_at %}{% when Some with (ts) %}{{ ts }}{% when None %}-{% endmatch %}</td>
|
||||||
|
<td>{% match jr.started_at %}{% when Some with (started) %}{% match jr.queued_at %}{% when Some with (queued) %}{{ started - queued }}ms{% when None %}-{% endmatch %}{% when None %}-{% endmatch %}</td>
|
||||||
|
<td>{% match jr.status %}{% when Some with (s) %}<span class="status status-{{ s.name_lowercase }}">{{ s.name }}</span>{% when None %}-{% endmatch %}</td>
|
||||||
|
</tr>
|
||||||
|
{% endfor %}
|
||||||
|
</tbody>
|
||||||
|
</table>
|
||||||
|
</div>
|
||||||
|
{% endif %}
|
||||||
|
|
||||||
|
{% if !want.derivative_wants.is_empty() %}
|
||||||
|
<div class="detail-section">
|
||||||
|
<h2>Fulfillment - Derivative Wants ({{ want.derivative_wants.len() }})</h2>
|
||||||
|
<table class="data-table">
|
||||||
|
<thead>
|
||||||
|
<tr>
|
||||||
|
<th>ID</th>
|
||||||
|
<th>Partitions</th>
|
||||||
|
<th>Status</th>
|
||||||
|
</tr>
|
||||||
|
</thead>
|
||||||
|
<tbody>
|
||||||
|
{% for dw in want.derivative_wants %}
|
||||||
|
<tr>
|
||||||
|
<td><a href="/wants/{{ dw.want_id }}">{{ dw.want_id }}</a></td>
|
||||||
|
<td>{% for p in dw.partitions %}{{ p.partition_ref }}{% if !loop.last %}, {% endif %}{% endfor %}</td>
|
||||||
|
<td>{% match dw.status %}{% when Some with (s) %}<span class="status status-{{ s.name|lower }}">{{ s.name }}</span>{% when None %}-{% endmatch %}</td>
|
||||||
|
</tr>
|
||||||
|
{% endfor %}
|
||||||
|
</tbody>
|
||||||
|
</table>
|
||||||
|
</div>
|
||||||
|
{% endif %}
|
||||||
|
|
||||||
{% call base::footer() %}
|
{% call base::footer() %}
|
||||||
|
|
|
||||||
12
docs/ideas/2025-11-30_wants-for-data-retention.md
Normal file
12
docs/ideas/2025-11-30_wants-for-data-retention.md
Normal file
|
|
@ -0,0 +1,12 @@
|
||||||
|
|
||||||
|
- You can't keep building partitions forever
|
||||||
|
- Space runs out for literal data, costs increase as data value decreases for old data
|
||||||
|
- Memory runs out for indexes over partitions, e.g. in databuild itself
|
||||||
|
- We could introduce vacuuming?
|
||||||
|
- We could introduce partition want expiry callbacks?
|
||||||
|
- Jobs and job runs as partition edge transitions?
|
||||||
|
- Do we even want to delete partition entries? Can just wait till this is a problem.
|
||||||
|
- Partition want expiry events also enable non-event level reaction, e.g. vacuum for all events between time T1 and T2.
|
||||||
|
- RISK! If partition to partition data deps (via jobs that change, etc) are not canonical/stable:
|
||||||
|
- We cannot assert the necessity of upstream partitions for anything longer than the initial job time to success (because it may have changed)
|
||||||
|
- To make this valuable, we need to be able to assume that the reads/data deps from a singular parameterized job run are durable, because then we can propagate want times and have a durable "why does this partition need to exist"
|
||||||
241
docs/ideas/event-sourced-cpn-framework.md
Normal file
241
docs/ideas/event-sourced-cpn-framework.md
Normal file
|
|
@ -0,0 +1,241 @@
|
||||||
|
# Event-Sourced CPN Framework
|
||||||
|
|
||||||
|
A vision for a Rust library/framework combining event sourcing, Colored Petri Net semantics, and compile-time safety for building correct distributed systems.
|
||||||
|
|
||||||
|
## The Problem
|
||||||
|
|
||||||
|
In highly connected applications with multiple entity types and relationships (like databuild's Wants, JobRuns, Partitions), developers face combinatorial complexity:
|
||||||
|
|
||||||
|
For each edge type between entities, you need:
|
||||||
|
1. Forward accessor
|
||||||
|
2. Inverse accessor (index)
|
||||||
|
3. Index maintenance on creation
|
||||||
|
4. Index maintenance on deletion
|
||||||
|
5. Consistency checks
|
||||||
|
6. Query patterns for traversal
|
||||||
|
|
||||||
|
As the number of entities and edges grows, this becomes:
|
||||||
|
- Hard to keep in your head
|
||||||
|
- Error-prone (forgot to update an index)
|
||||||
|
- Lots of boilerplate
|
||||||
|
- Testing burden for plumbing rather than business logic
|
||||||
|
|
||||||
|
The temptation is to "throw hands up" and use SQL with foreign keys, or accept eventual consistency. But this sacrifices the compile-time guarantees Rust can provide.
|
||||||
|
|
||||||
|
## The Vision
|
||||||
|
|
||||||
|
A framework where developers declare:
|
||||||
|
- **Entities** with their valid states (state machines)
|
||||||
|
- **Edges** between entities (typed, directional, with cardinality)
|
||||||
|
- **Transitions** (what state changes are valid, and when)
|
||||||
|
|
||||||
|
And the framework provides:
|
||||||
|
- Auto-generated accessors (both directions)
|
||||||
|
- Auto-maintained indexes
|
||||||
|
- Compile-time invalid transition errors
|
||||||
|
- Runtime referential integrity (fail-fast or transactional)
|
||||||
|
- Event log as source of truth with replay capability
|
||||||
|
- Potential automatic concurrency from CPN place-disjointness
|
||||||
|
|
||||||
|
## Why
|
||||||
|
|
||||||
|
### Correctness Guarantees
|
||||||
|
|
||||||
|
- **Compile-time**: Invalid state transitions are type errors
|
||||||
|
- **Compile-time**: Edge definitions guarantee bidirectional navigability
|
||||||
|
- **Runtime**: Referential integrity violations detected immediately
|
||||||
|
- **Result**: "If it compiles and the event log replays, the state is consistent"
|
||||||
|
|
||||||
|
### Performance "For Free"
|
||||||
|
|
||||||
|
- Indexes auto-maintained as edges are created/destroyed
|
||||||
|
- No query planning needed - traversal patterns known at compile time
|
||||||
|
- Potential: CPN place-disjointness → automatic safe concurrency
|
||||||
|
|
||||||
|
### Developer Experience
|
||||||
|
|
||||||
|
- Declare entities, states, edges, transitions
|
||||||
|
- Library generates: accessors, inverse indexes, transition methods, consistency checks
|
||||||
|
- Focus on *what* not *how* - the plumbing disappears
|
||||||
|
- Still Rust: escape hatch to custom logic when needed
|
||||||
|
|
||||||
|
### Testing Burden Reduction
|
||||||
|
|
||||||
|
- No tests for "did I update the index correctly"
|
||||||
|
- No tests for "can I traverse this relationship backwards"
|
||||||
|
- Focus tests on business logic, not graph bookkeeping
|
||||||
|
|
||||||
|
## How
|
||||||
|
|
||||||
|
### Foundations
|
||||||
|
|
||||||
|
- **Colored Petri Nets** for state machine composition semantics
|
||||||
|
- **Typestate pattern** for compile-time transition validity
|
||||||
|
- **Event sourcing** for persistence and replay
|
||||||
|
|
||||||
|
### Implementation Approach
|
||||||
|
|
||||||
|
Declarative DSL or proc macros for entity/edge/transition definitions:
|
||||||
|
|
||||||
|
```rust
|
||||||
|
// Hypothetical syntax
|
||||||
|
entity! {
|
||||||
|
Want {
|
||||||
|
states: [New, Idle, Building, Successful, Failed, Canceled],
|
||||||
|
transitions: [
|
||||||
|
New -> Idle,
|
||||||
|
New -> Building,
|
||||||
|
Idle -> Building,
|
||||||
|
Building -> Successful,
|
||||||
|
Building -> Failed,
|
||||||
|
// ...
|
||||||
|
]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
edge! {
|
||||||
|
servicing_wants: JobRun -> many Want,
|
||||||
|
built_by: Partition -> one JobRun,
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
Code generation produces:
|
||||||
|
- Entity structs with state type parameters
|
||||||
|
- Edge storage with auto-maintained inverses
|
||||||
|
- Transition methods that enforce valid source states
|
||||||
|
- Query methods for traversal in both directions
|
||||||
|
|
||||||
|
### The Graph Model
|
||||||
|
|
||||||
|
- Entities are nodes (with state)
|
||||||
|
- Edges are typed, directional, with cardinality (one/many)
|
||||||
|
- Both directions always queryable
|
||||||
|
- Edge creation/deletion is transactional within a step
|
||||||
|
|
||||||
|
### Entry Point
|
||||||
|
|
||||||
|
Single `step(event) -> Result<(), StepError>` that:
|
||||||
|
1. Validates the event against current state
|
||||||
|
2. Applies state transitions
|
||||||
|
3. Updates all affected indexes
|
||||||
|
4. Returns success or rolls back
|
||||||
|
|
||||||
|
## Transactionality
|
||||||
|
|
||||||
|
### Beyond Fail-Fast
|
||||||
|
|
||||||
|
Instead of panicking on consistency violations, support transactional semantics:
|
||||||
|
|
||||||
|
```rust
|
||||||
|
// Infallible (panics on error)
|
||||||
|
state.step(event);
|
||||||
|
|
||||||
|
// Fallible (returns error, state unchanged on failure)
|
||||||
|
state.try_step(event) -> Result<(), StepError>;
|
||||||
|
|
||||||
|
// Explicit transaction (for multi-event atomicity)
|
||||||
|
let txn = state.begin();
|
||||||
|
txn.apply(event1)?;
|
||||||
|
txn.apply(event2)?;
|
||||||
|
txn.commit(); // or rollback on drop
|
||||||
|
```
|
||||||
|
|
||||||
|
### What This Enables
|
||||||
|
|
||||||
|
1. **Local atomicity**: A single event either fully applies or doesn't - no partial states
|
||||||
|
|
||||||
|
2. **Distributed coordination**: If `step` can return `Err` instead of panicking:
|
||||||
|
- Try to apply an event
|
||||||
|
- If it fails, coordinate with other systems before retrying
|
||||||
|
- Implement saga patterns, 2PC, etc.
|
||||||
|
|
||||||
|
3. **Speculative execution**: "What if I applied this event?" without committing
|
||||||
|
- Useful for validation, dry-runs, conflict detection
|
||||||
|
|
||||||
|
4. **Optimistic concurrency**:
|
||||||
|
- Multiple workers try to apply events concurrently
|
||||||
|
- Conflicts detected and rolled back
|
||||||
|
- Retry with updated state
|
||||||
|
|
||||||
|
### Implementation Options
|
||||||
|
|
||||||
|
1. **Copy-on-write / snapshot**: Clone state, apply to clone, swap on success
|
||||||
|
- Simple but memory-heavy for large state
|
||||||
|
|
||||||
|
2. **Command pattern / undo log**: Record inverse operations, replay backwards on rollback
|
||||||
|
- More complex, but efficient for small changes to large state
|
||||||
|
|
||||||
|
3. **MVCC-style**: Version all entities, only "commit" versions on success
|
||||||
|
- Most sophisticated, enables concurrent reads during transaction
|
||||||
|
|
||||||
|
## Relationship to Datomic
|
||||||
|
|
||||||
|
[Datomic](https://docs.datomic.com/datomic-overview.html) is a distributed database built on similar principles that validates many of these ideas in production:
|
||||||
|
|
||||||
|
### Shared Concepts
|
||||||
|
|
||||||
|
| Concept | Datomic | This Framework |
|
||||||
|
|---------|---------|----------------|
|
||||||
|
| Immutable facts | Datoms (E-A-V-T tuples) | BEL events |
|
||||||
|
| Time travel | `as-of` queries | Event replay |
|
||||||
|
| Speculative execution | [`d/with`](https://docs.datomic.com/transactions/transaction-processing.html) | `try_step()` / transactions |
|
||||||
|
| Atomic commits | `d/transact` = `d/with` + durable swap | `step()` = validate + apply + persist |
|
||||||
|
| Transaction-time validation | [Transaction functions](https://docs.datomic.com/transactions/transaction-functions.html) with `db-before` | Transition guards |
|
||||||
|
| Post-transaction validation | [Entity specs](https://docs.datomic.com/transactions/model.html) with `db-after` | Invariant checks |
|
||||||
|
| Single writer | Transactor serializes all writes | Single `step()` entry point |
|
||||||
|
| Horizontal read scaling | Peers cache and query locally | Immutable state snapshots |
|
||||||
|
|
||||||
|
### Datomic's Speculative Writes
|
||||||
|
|
||||||
|
Datomic's `d/with` is particularly relevant - it's a [pure function](https://vvvvalvalval.github.io/posts/2018-11-12-datomic-event-sourcing-without-the-hassle.html) that takes a database value and proposed facts, returning a new database value *without persisting*. This enables:
|
||||||
|
|
||||||
|
- Testing transactions without mutation
|
||||||
|
- Composing transaction data before committing
|
||||||
|
- [Enforcing invariants](https://stackoverflow.com/questions/48268887/how-to-prevent-transactions-from-violating-application-invariants-in-datomic) by speculatively applying, checking, then committing or aborting
|
||||||
|
- Development against production data safely (via libraries like Datomock)
|
||||||
|
|
||||||
|
### What Datomic Doesn't Provide
|
||||||
|
|
||||||
|
- **CPN state machine semantics**: Typed transitions between entity states
|
||||||
|
- **Compile-time transition validity**: Invalid transitions caught by the type system
|
||||||
|
- **Auto-generated bidirectional indexes**: Declared edges automatically traversable both ways
|
||||||
|
- **Rust**: Memory safety, zero-cost abstractions, embeddable
|
||||||
|
|
||||||
|
The vision here is essentially: *Datomic's transaction model + CPN state machines + Rust compile-time safety*
|
||||||
|
|
||||||
|
## Open Questions
|
||||||
|
|
||||||
|
- How to express transition guards (conditions beyond "in state X")?
|
||||||
|
- How to handle edges to entities that don't exist yet (forward references)?
|
||||||
|
- Serialization format for the event log?
|
||||||
|
- How much CPN formalism to expose vs. hide?
|
||||||
|
- What's the right granularity for "places" in the CPN model?
|
||||||
|
- How does this interact with async/distributed systems?
|
||||||
|
|
||||||
|
## Potential Names
|
||||||
|
|
||||||
|
Something evoking: event-sourced + graph + state machines + Rust
|
||||||
|
|
||||||
|
- `petri-graph`
|
||||||
|
- `ironweave` (iron = Rust, weave = connected graph)
|
||||||
|
- `factforge`
|
||||||
|
- `datumflow`
|
||||||
|
|
||||||
|
## Prior Art to Investigate
|
||||||
|
|
||||||
|
- Datomic (Clojure, distributed immutable database)
|
||||||
|
- Bevy ECS (Rust, entity-component-system with events)
|
||||||
|
- CPN Tools (Petri net modeling/simulation)
|
||||||
|
- Diesel / SeaORM (Rust, compile-time SQL checking)
|
||||||
|
- EventStoreDB (event sourcing infrastructure)
|
||||||
|
|
||||||
|
## Next Steps
|
||||||
|
|
||||||
|
This document captures the "why" and "how" at a conceptual level. To validate:
|
||||||
|
|
||||||
|
1. Prototype the macro/DSL syntax for a simple 2-3 entity system
|
||||||
|
2. Implement auto-indexed bidirectional edges
|
||||||
|
3. Implement typestate transitions
|
||||||
|
4. Add speculative execution (`try_step`)
|
||||||
|
5. Benchmark against hand-written equivalent
|
||||||
|
6. Evaluate ergonomics in real use (databuild as first consumer)
|
||||||
7
docs/narrative/what-llms-dont-do.md
Normal file
7
docs/narrative/what-llms-dont-do.md
Normal file
|
|
@ -0,0 +1,7 @@
|
||||||
|
|
||||||
|
# What LLMs Don't Do
|
||||||
|
|
||||||
|
- Create and cultivate technical strategy
|
||||||
|
- Don't have a specific vision of the organizing formalization of the problem + the technical solution
|
||||||
|
- Adhere to technical strategy
|
||||||
|
- Please please please just read the relevant docs!
|
||||||
230
docs/plans/detail-lineage.md
Normal file
230
docs/plans/detail-lineage.md
Normal file
|
|
@ -0,0 +1,230 @@
|
||||||
|
# Detail & Lineage Views
|
||||||
|
|
||||||
|
## Vision
|
||||||
|
|
||||||
|
Provide rich, navigable views into databuild's execution history that answer operational questions:
|
||||||
|
|
||||||
|
- **"What work was done to fulfill this want?"** - The full DAG of partitions built and jobs run
|
||||||
|
- **"Where did this data come from?"** - Trace a partition's lineage back through its inputs
|
||||||
|
- **"What downstream data uses this?"** - Understand impact before tainting or debugging staleness
|
||||||
|
|
||||||
|
## Three Distinct Views
|
||||||
|
|
||||||
|
### 1. Want Fulfillment View
|
||||||
|
|
||||||
|
Shows the work tree rooted at a want: all partitions built, jobs run, and derivative wants spawned to fulfill it.
|
||||||
|
|
||||||
|
```
|
||||||
|
W-001 "data/gamma" [Successful]
|
||||||
|
│
|
||||||
|
├── data/gamma [Live, uuid:abc]
|
||||||
|
│ └── JR-789 [Succeeded]
|
||||||
|
│ ├── read: data/beta [Live, uuid:def]
|
||||||
|
│ └── read: data/alpha [Live, uuid:ghi]
|
||||||
|
│
|
||||||
|
└── derivative: W-002 "data/beta" [Successful]
|
||||||
|
│ └── triggered by: JR-456 dep-miss
|
||||||
|
│
|
||||||
|
└── data/beta [Live, uuid:def]
|
||||||
|
└── JR-456 [DepMiss → retry → Succeeded]
|
||||||
|
└── read: data/alpha [Live, uuid:ghi]
|
||||||
|
```
|
||||||
|
|
||||||
|
Key insight: This shows **specific partition instances** (by UUID), not just refs. A want's fulfillment is a concrete snapshot of what was built.
|
||||||
|
|
||||||
|
### 2. Partition Lineage View
|
||||||
|
|
||||||
|
The data flow graph: partition ↔ job_run alternating. Navigable upstream (inputs) and downstream (consumers).
|
||||||
|
|
||||||
|
```
|
||||||
|
UPSTREAM
|
||||||
|
│
|
||||||
|
┌────────────┼────────────┐
|
||||||
|
▼ ▼ ▼
|
||||||
|
[data/a] [data/b] [data/c]
|
||||||
|
│ │ │
|
||||||
|
└────────────┼────────────┘
|
||||||
|
▼
|
||||||
|
JR-xyz [Succeeded]
|
||||||
|
│
|
||||||
|
▼
|
||||||
|
══════════════════
|
||||||
|
║ data/beta ║ ← FOCUS
|
||||||
|
║ [Live] ║
|
||||||
|
══════════════════
|
||||||
|
│
|
||||||
|
▼
|
||||||
|
JR-abc [Running]
|
||||||
|
│
|
||||||
|
┌────────────┼────────────┐
|
||||||
|
▼ ▼ ▼
|
||||||
|
[data/x] [data/y] [data/z]
|
||||||
|
│
|
||||||
|
DOWNSTREAM
|
||||||
|
```
|
||||||
|
|
||||||
|
This view answers: "What data flows into/out of this partition?" Click to navigate.
|
||||||
|
|
||||||
|
### 3. JobRun Detail View
|
||||||
|
|
||||||
|
Not a graph - just the immediate context of a single job execution:
|
||||||
|
|
||||||
|
- **Scheduled for**: Which want(s) triggered this job
|
||||||
|
- **Read**: Input partitions (with UUIDs - the specific versions read)
|
||||||
|
- **Wrote**: Output partitions (with UUIDs)
|
||||||
|
- **Status history**: Queued → Running → Succeeded/Failed/DepMiss
|
||||||
|
- **If DepMiss**: Which derivative wants were spawned
|
||||||
|
|
||||||
|
## Data Requirements
|
||||||
|
|
||||||
|
### Track read_deps on success
|
||||||
|
|
||||||
|
Currently only captured on dep-miss. Need to extend `JobRunSuccessEventV1`:
|
||||||
|
|
||||||
|
```protobuf
|
||||||
|
message JobRunSuccessEventV1 {
|
||||||
|
string job_run_id = 1;
|
||||||
|
repeated ReadDeps read_deps = 2; // NEW
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
### Inverted consumer index
|
||||||
|
|
||||||
|
To answer "what reads this partition", need:
|
||||||
|
|
||||||
|
```rust
|
||||||
|
partition_consumers: BTreeMap<Uuid, Vec<(Uuid, String)>> // input_uuid → (output_uuid, job_run_id)
|
||||||
|
```
|
||||||
|
|
||||||
|
Indexed by UUID (not ref) because partition refs get reused across rebuilds, but UUIDs are immutable per instance. This preserves historical lineage correctly.
|
||||||
|
|
||||||
|
Built from read_deps when processing JobRunSuccessEventV1.
|
||||||
|
|
||||||
|
## Design Decisions
|
||||||
|
|
||||||
|
1. **Retries**: List all job runs triggered by a want, collapsing retries in the UI (expandable)
|
||||||
|
2. **Lineage UUIDs**: Resolve partition refs to canonical UUIDs at job success time (jobs don't need to know about UUIDs)
|
||||||
|
3. **High fan-out**: Truncate to N items with "+X more" expansion
|
||||||
|
4. **Consumer index by UUID**: Index consumers by partition UUID (not ref) since refs get reused across rebuilds but UUIDs are immutable per instance
|
||||||
|
5. **Job run as lineage source of truth**: Partition details don't duplicate upstream info - they reference their builder job run, which holds the read_deps
|
||||||
|
|
||||||
|
## API Response Pattern
|
||||||
|
|
||||||
|
Detail and list endpoints return a wrapper with the primary data plus a shared index of related entities:
|
||||||
|
|
||||||
|
```protobuf
|
||||||
|
message GetJobRunResponse {
|
||||||
|
JobRunDetail data = 1;
|
||||||
|
RelatedEntities index = 2;
|
||||||
|
}
|
||||||
|
|
||||||
|
message ListJobRunsResponse {
|
||||||
|
repeated JobRunDetail data = 1;
|
||||||
|
RelatedEntities index = 2; // shared across all items
|
||||||
|
}
|
||||||
|
|
||||||
|
message RelatedEntities {
|
||||||
|
map<string, PartitionDetail> partitions = 1;
|
||||||
|
map<string, JobRunDetail> job_runs = 2;
|
||||||
|
map<string, WantDetail> wants = 3;
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
**Why this pattern:**
|
||||||
|
- **No recursion** - Detail types stay flat, don't embed each other
|
||||||
|
- **Deduplication** - Each entity appears once in the index, even if referenced by multiple items in `data`
|
||||||
|
- **O(1) lookup** - Templates access `index.partitions["data/beta"]` directly
|
||||||
|
- **Composable** - Same pattern works for single-item and list endpoints
|
||||||
|
|
||||||
|
## Implementation Plan
|
||||||
|
|
||||||
|
### ✅ Phase 1: Data Model (Complete)
|
||||||
|
|
||||||
|
**1.1 Extend JobRunSuccessEventV1**
|
||||||
|
|
||||||
|
```protobuf
|
||||||
|
message JobRunSuccessEventV1 {
|
||||||
|
string job_run_id = 1;
|
||||||
|
repeated ReadDeps read_deps = 2; // preserves impacted→read relationships
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
**1.2 Extend SucceededState to store resolved UUIDs**
|
||||||
|
|
||||||
|
```rust
|
||||||
|
pub struct SucceededState {
|
||||||
|
pub succeeded_at: u64,
|
||||||
|
pub read_deps: Vec<ReadDeps>, // from event
|
||||||
|
pub read_partition_uuids: BTreeMap<String, Uuid>, // ref → UUID at read time
|
||||||
|
pub wrote_partition_uuids: BTreeMap<String, Uuid>, // ref → UUID (from building_partitions)
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
UUIDs resolved by looking up canonical partitions when processing success event.
|
||||||
|
|
||||||
|
**1.3 Add consumer index to BuildState**
|
||||||
|
|
||||||
|
```rust
|
||||||
|
// input_uuid → list of (output_uuid, job_run_id)
|
||||||
|
partition_consumers: BTreeMap<Uuid, Vec<(Uuid, String)>>
|
||||||
|
```
|
||||||
|
|
||||||
|
Populated from `read_deps` when processing JobRunSuccessEventV1. Uses UUIDs (not refs) to preserve historical lineage across partition rebuilds.
|
||||||
|
|
||||||
|
### ✅ Phase 2: API Response Pattern (Complete)
|
||||||
|
|
||||||
|
**2.1 RelatedEntities wrapper**
|
||||||
|
|
||||||
|
Added `RelatedEntities` message and `index` field to all Get*/List* responses.
|
||||||
|
|
||||||
|
**2.2 HasRelatedIds trait**
|
||||||
|
|
||||||
|
Implemented trait for Want, JobRun, Partition that returns the IDs of related entities. Query layer uses this to build the index.
|
||||||
|
|
||||||
|
**2.3 Query methods**
|
||||||
|
|
||||||
|
Added `*_with_index()` methods that collect related IDs via the trait and resolve them to full entity details.
|
||||||
|
|
||||||
|
### ✅ Phase 3: Job Integration (Complete)
|
||||||
|
|
||||||
|
Jobs already emit `DATABUILD_DEP_READ_JSON` and the full pipeline is wired up:
|
||||||
|
|
||||||
|
1. **Job execution** (`job_run.rs`): `SubProcessBackend::poll` parses `DATABUILD_DEP_READ_JSON` lines from stdout and stores in `SubProcessCompleted.read_deps`
|
||||||
|
2. **Event creation** (`job_run.rs`): `to_event()` creates `JobRunSuccessEventV1` with `read_deps`
|
||||||
|
3. **Event handling** (`event_handlers.rs`): `handle_job_run_success()` resolves `read_partition_uuids` and `wrote_partition_uuids`, populates `partition_consumers` index
|
||||||
|
4. **API serialization** (`job_run_state.rs`): `to_detail()` includes `read_deps`, `read_partition_uuids`, `wrote_partition_uuids` in `JobRunDetail`
|
||||||
|
|
||||||
|
### ✅ Phase 4: Frontend (Complete)
|
||||||
|
|
||||||
|
**4.1 JobRun detail page**
|
||||||
|
|
||||||
|
Added to `job_runs/detail.html`:
|
||||||
|
- "Read Partitions" section showing partition refs with UUIDs (linked to partition detail)
|
||||||
|
- "Wrote Partitions" section showing partition refs with UUIDs (linked to partition detail)
|
||||||
|
- "Derivative Wants" section showing wants spawned by dep-miss (linked to want detail)
|
||||||
|
|
||||||
|
Extended `JobRunDetailView` with:
|
||||||
|
- `read_deps: Vec<ReadDepsView>` - impacted→read dependency relationships
|
||||||
|
- `read_partitions: Vec<PartitionRefWithUuidView>` - input partitions with UUIDs
|
||||||
|
- `wrote_partitions: Vec<PartitionRefWithUuidView>` - output partitions with UUIDs
|
||||||
|
- `derivative_want_ids: Vec<String>` - derivative wants from dep-miss
|
||||||
|
|
||||||
|
**4.2 Partition detail page**
|
||||||
|
|
||||||
|
Added to `partitions/detail.html`:
|
||||||
|
- "Lineage - Built By" section showing the builder job run (linked to job run detail for upstream lineage)
|
||||||
|
- "Lineage - Downstream Consumers" section showing UUIDs of downstream partitions
|
||||||
|
|
||||||
|
Extended `PartitionDetailView` with:
|
||||||
|
- `built_by_job_run_id: Option<String>` - job run that built this partition
|
||||||
|
- `downstream_partition_uuids: Vec<String>` - downstream consumers from index
|
||||||
|
|
||||||
|
**4.3 Want detail page**
|
||||||
|
|
||||||
|
Added to `wants/detail.html`:
|
||||||
|
- "Fulfillment - Job Runs" section listing all job runs that serviced this want
|
||||||
|
- "Fulfillment - Derivative Wants" section listing derivative wants spawned by dep-misses
|
||||||
|
|
||||||
|
Extended `WantDetailView` with:
|
||||||
|
- `job_run_ids: Vec<String>` - all job runs that serviced this want
|
||||||
|
- `derivative_want_ids: Vec<String>` - derivative wants spawned
|
||||||
|
|
@ -3,7 +3,7 @@
|
||||||
"jobs": [
|
"jobs": [
|
||||||
{
|
{
|
||||||
"label": "//examples/multihop:job_alpha",
|
"label": "//examples/multihop:job_alpha",
|
||||||
"entrypoint": "./examples/multihop/job_alpha.sh",
|
"entrypoint": "./job_alpha.sh",
|
||||||
"environment": {
|
"environment": {
|
||||||
"JOB_NAME": "alpha"
|
"JOB_NAME": "alpha"
|
||||||
},
|
},
|
||||||
|
|
@ -11,7 +11,7 @@
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"label": "//examples/multihop:job_beta",
|
"label": "//examples/multihop:job_beta",
|
||||||
"entrypoint": "./examples/multihop/job_beta.sh",
|
"entrypoint": "./job_beta.sh",
|
||||||
"environment": {
|
"environment": {
|
||||||
"JOB_NAME": "beta"
|
"JOB_NAME": "beta"
|
||||||
},
|
},
|
||||||
|
|
|
||||||
|
|
@ -10,8 +10,9 @@
|
||||||
"https://bcr.bazel.build/modules/abseil-cpp/20230802.1/MODULE.bazel": "fa92e2eb41a04df73cdabeec37107316f7e5272650f81d6cc096418fe647b915",
|
"https://bcr.bazel.build/modules/abseil-cpp/20230802.1/MODULE.bazel": "fa92e2eb41a04df73cdabeec37107316f7e5272650f81d6cc096418fe647b915",
|
||||||
"https://bcr.bazel.build/modules/abseil-cpp/20240116.1/MODULE.bazel": "37bcdb4440fbb61df6a1c296ae01b327f19e9bb521f9b8e26ec854b6f97309ed",
|
"https://bcr.bazel.build/modules/abseil-cpp/20240116.1/MODULE.bazel": "37bcdb4440fbb61df6a1c296ae01b327f19e9bb521f9b8e26ec854b6f97309ed",
|
||||||
"https://bcr.bazel.build/modules/abseil-cpp/20240116.1/source.json": "9be551b8d4e3ef76875c0d744b5d6a504a27e3ae67bc6b28f46415fd2d2957da",
|
"https://bcr.bazel.build/modules/abseil-cpp/20240116.1/source.json": "9be551b8d4e3ef76875c0d744b5d6a504a27e3ae67bc6b28f46415fd2d2957da",
|
||||||
"https://bcr.bazel.build/modules/apple_support/1.17.1/MODULE.bazel": "655c922ab1209978a94ef6ca7d9d43e940cd97d9c172fb55f94d91ac53f8610b",
|
"https://bcr.bazel.build/modules/apple_support/1.23.1/MODULE.bazel": "53763fed456a968cf919b3240427cf3a9d5481ec5466abc9d5dc51bc70087442",
|
||||||
"https://bcr.bazel.build/modules/apple_support/1.17.1/source.json": "6b2b8c74d14e8d485528a938e44bdb72a5ba17632b9e14ef6e68a5ee96c8347f",
|
"https://bcr.bazel.build/modules/apple_support/1.24.1/MODULE.bazel": "f46e8ddad60aef170ee92b2f3d00ef66c147ceafea68b6877cb45bd91737f5f8",
|
||||||
|
"https://bcr.bazel.build/modules/apple_support/1.24.1/source.json": "cf725267cbacc5f028ef13bb77e7f2c2e0066923a4dab1025e4a0511b1ed258a",
|
||||||
"https://bcr.bazel.build/modules/aspect_bazel_lib/2.14.0/MODULE.bazel": "2b31ffcc9bdc8295b2167e07a757dbbc9ac8906e7028e5170a3708cecaac119f",
|
"https://bcr.bazel.build/modules/aspect_bazel_lib/2.14.0/MODULE.bazel": "2b31ffcc9bdc8295b2167e07a757dbbc9ac8906e7028e5170a3708cecaac119f",
|
||||||
"https://bcr.bazel.build/modules/aspect_bazel_lib/2.14.0/source.json": "0cf1826853b0bef8b5cd19c0610d717500f5521aa2b38b72b2ec302ac5e7526c",
|
"https://bcr.bazel.build/modules/aspect_bazel_lib/2.14.0/source.json": "0cf1826853b0bef8b5cd19c0610d717500f5521aa2b38b72b2ec302ac5e7526c",
|
||||||
"https://bcr.bazel.build/modules/aspect_bazel_lib/2.7.2/MODULE.bazel": "780d1a6522b28f5edb7ea09630748720721dfe27690d65a2d33aa7509de77e07",
|
"https://bcr.bazel.build/modules/aspect_bazel_lib/2.7.2/MODULE.bazel": "780d1a6522b28f5edb7ea09630748720721dfe27690d65a2d33aa7509de77e07",
|
||||||
|
|
@ -31,8 +32,11 @@
|
||||||
"https://bcr.bazel.build/modules/bazel_features/1.18.0/MODULE.bazel": "1be0ae2557ab3a72a57aeb31b29be347bcdc5d2b1eb1e70f39e3851a7e97041a",
|
"https://bcr.bazel.build/modules/bazel_features/1.18.0/MODULE.bazel": "1be0ae2557ab3a72a57aeb31b29be347bcdc5d2b1eb1e70f39e3851a7e97041a",
|
||||||
"https://bcr.bazel.build/modules/bazel_features/1.19.0/MODULE.bazel": "59adcdf28230d220f0067b1f435b8537dd033bfff8db21335ef9217919c7fb58",
|
"https://bcr.bazel.build/modules/bazel_features/1.19.0/MODULE.bazel": "59adcdf28230d220f0067b1f435b8537dd033bfff8db21335ef9217919c7fb58",
|
||||||
"https://bcr.bazel.build/modules/bazel_features/1.21.0/MODULE.bazel": "675642261665d8eea09989aa3b8afb5c37627f1be178382c320d1b46afba5e3b",
|
"https://bcr.bazel.build/modules/bazel_features/1.21.0/MODULE.bazel": "675642261665d8eea09989aa3b8afb5c37627f1be178382c320d1b46afba5e3b",
|
||||||
|
"https://bcr.bazel.build/modules/bazel_features/1.27.0/MODULE.bazel": "621eeee06c4458a9121d1f104efb80f39d34deff4984e778359c60eaf1a8cb65",
|
||||||
|
"https://bcr.bazel.build/modules/bazel_features/1.28.0/MODULE.bazel": "4b4200e6cbf8fa335b2c3f43e1d6ef3e240319c33d43d60cc0fbd4b87ece299d",
|
||||||
"https://bcr.bazel.build/modules/bazel_features/1.30.0/MODULE.bazel": "a14b62d05969a293b80257e72e597c2da7f717e1e69fa8b339703ed6731bec87",
|
"https://bcr.bazel.build/modules/bazel_features/1.30.0/MODULE.bazel": "a14b62d05969a293b80257e72e597c2da7f717e1e69fa8b339703ed6731bec87",
|
||||||
"https://bcr.bazel.build/modules/bazel_features/1.30.0/source.json": "b07e17f067fe4f69f90b03b36ef1e08fe0d1f3cac254c1241a1818773e3423bc",
|
"https://bcr.bazel.build/modules/bazel_features/1.32.0/MODULE.bazel": "095d67022a58cb20f7e20e1aefecfa65257a222c18a938e2914fd257b5f1ccdc",
|
||||||
|
"https://bcr.bazel.build/modules/bazel_features/1.32.0/source.json": "2546c766986a6541f0bacd3e8542a1f621e2b14a80ea9e88c6f89f7eedf64ae1",
|
||||||
"https://bcr.bazel.build/modules/bazel_features/1.4.1/MODULE.bazel": "e45b6bb2350aff3e442ae1111c555e27eac1d915e77775f6fdc4b351b758b5d7",
|
"https://bcr.bazel.build/modules/bazel_features/1.4.1/MODULE.bazel": "e45b6bb2350aff3e442ae1111c555e27eac1d915e77775f6fdc4b351b758b5d7",
|
||||||
"https://bcr.bazel.build/modules/bazel_features/1.9.0/MODULE.bazel": "885151d58d90d8d9c811eb75e3288c11f850e1d6b481a8c9f766adee4712358b",
|
"https://bcr.bazel.build/modules/bazel_features/1.9.0/MODULE.bazel": "885151d58d90d8d9c811eb75e3288c11f850e1d6b481a8c9f766adee4712358b",
|
||||||
"https://bcr.bazel.build/modules/bazel_features/1.9.1/MODULE.bazel": "8f679097876a9b609ad1f60249c49d68bfab783dd9be012faf9d82547b14815a",
|
"https://bcr.bazel.build/modules/bazel_features/1.9.1/MODULE.bazel": "8f679097876a9b609ad1f60249c49d68bfab783dd9be012faf9d82547b14815a",
|
||||||
|
|
@ -48,7 +52,8 @@
|
||||||
"https://bcr.bazel.build/modules/bazel_skylib/1.7.0/MODULE.bazel": "0db596f4563de7938de764cc8deeabec291f55e8ec15299718b93c4423e9796d",
|
"https://bcr.bazel.build/modules/bazel_skylib/1.7.0/MODULE.bazel": "0db596f4563de7938de764cc8deeabec291f55e8ec15299718b93c4423e9796d",
|
||||||
"https://bcr.bazel.build/modules/bazel_skylib/1.7.1/MODULE.bazel": "3120d80c5861aa616222ec015332e5f8d3171e062e3e804a2a0253e1be26e59b",
|
"https://bcr.bazel.build/modules/bazel_skylib/1.7.1/MODULE.bazel": "3120d80c5861aa616222ec015332e5f8d3171e062e3e804a2a0253e1be26e59b",
|
||||||
"https://bcr.bazel.build/modules/bazel_skylib/1.8.1/MODULE.bazel": "88ade7293becda963e0e3ea33e7d54d3425127e0a326e0d17da085a5f1f03ff6",
|
"https://bcr.bazel.build/modules/bazel_skylib/1.8.1/MODULE.bazel": "88ade7293becda963e0e3ea33e7d54d3425127e0a326e0d17da085a5f1f03ff6",
|
||||||
"https://bcr.bazel.build/modules/bazel_skylib/1.8.1/source.json": "7ebaefba0b03efe59cac88ed5bbc67bcf59a3eff33af937345ede2a38b2d368a",
|
"https://bcr.bazel.build/modules/bazel_skylib/1.8.2/MODULE.bazel": "69ad6927098316848b34a9142bcc975e018ba27f08c4ff403f50c1b6e646ca67",
|
||||||
|
"https://bcr.bazel.build/modules/bazel_skylib/1.8.2/source.json": "34a3c8bcf233b835eb74be9d628899bb32999d3e0eadef1947a0a562a2b16ffb",
|
||||||
"https://bcr.bazel.build/modules/buildozer/7.1.2/MODULE.bazel": "2e8dd40ede9c454042645fd8d8d0cd1527966aa5c919de86661e62953cd73d84",
|
"https://bcr.bazel.build/modules/buildozer/7.1.2/MODULE.bazel": "2e8dd40ede9c454042645fd8d8d0cd1527966aa5c919de86661e62953cd73d84",
|
||||||
"https://bcr.bazel.build/modules/buildozer/7.1.2/source.json": "c9028a501d2db85793a6996205c8de120944f50a0d570438fcae0457a5f9d1f8",
|
"https://bcr.bazel.build/modules/buildozer/7.1.2/source.json": "c9028a501d2db85793a6996205c8de120944f50a0d570438fcae0457a5f9d1f8",
|
||||||
"https://bcr.bazel.build/modules/google_benchmark/1.8.2/MODULE.bazel": "a70cf1bba851000ba93b58ae2f6d76490a9feb74192e57ab8e8ff13c34ec50cb",
|
"https://bcr.bazel.build/modules/google_benchmark/1.8.2/MODULE.bazel": "a70cf1bba851000ba93b58ae2f6d76490a9feb74192e57ab8e8ff13c34ec50cb",
|
||||||
|
|
@ -61,13 +66,14 @@
|
||||||
"https://bcr.bazel.build/modules/libpfm/4.11.0/MODULE.bazel": "45061ff025b301940f1e30d2c16bea596c25b176c8b6b3087e92615adbd52902",
|
"https://bcr.bazel.build/modules/libpfm/4.11.0/MODULE.bazel": "45061ff025b301940f1e30d2c16bea596c25b176c8b6b3087e92615adbd52902",
|
||||||
"https://bcr.bazel.build/modules/platforms/0.0.10/MODULE.bazel": "8cb8efaf200bdeb2150d93e162c40f388529a25852b332cec879373771e48ed5",
|
"https://bcr.bazel.build/modules/platforms/0.0.10/MODULE.bazel": "8cb8efaf200bdeb2150d93e162c40f388529a25852b332cec879373771e48ed5",
|
||||||
"https://bcr.bazel.build/modules/platforms/0.0.11/MODULE.bazel": "0daefc49732e227caa8bfa834d65dc52e8cc18a2faf80df25e8caea151a9413f",
|
"https://bcr.bazel.build/modules/platforms/0.0.11/MODULE.bazel": "0daefc49732e227caa8bfa834d65dc52e8cc18a2faf80df25e8caea151a9413f",
|
||||||
"https://bcr.bazel.build/modules/platforms/0.0.11/source.json": "f7e188b79ebedebfe75e9e1d098b8845226c7992b307e28e1496f23112e8fc29",
|
|
||||||
"https://bcr.bazel.build/modules/platforms/0.0.4/MODULE.bazel": "9b328e31ee156f53f3c416a64f8491f7eb731742655a47c9eec4703a71644aee",
|
"https://bcr.bazel.build/modules/platforms/0.0.4/MODULE.bazel": "9b328e31ee156f53f3c416a64f8491f7eb731742655a47c9eec4703a71644aee",
|
||||||
"https://bcr.bazel.build/modules/platforms/0.0.5/MODULE.bazel": "5733b54ea419d5eaf7997054bb55f6a1d0b5ff8aedf0176fef9eea44f3acda37",
|
"https://bcr.bazel.build/modules/platforms/0.0.5/MODULE.bazel": "5733b54ea419d5eaf7997054bb55f6a1d0b5ff8aedf0176fef9eea44f3acda37",
|
||||||
"https://bcr.bazel.build/modules/platforms/0.0.6/MODULE.bazel": "ad6eeef431dc52aefd2d77ed20a4b353f8ebf0f4ecdd26a807d2da5aa8cd0615",
|
"https://bcr.bazel.build/modules/platforms/0.0.6/MODULE.bazel": "ad6eeef431dc52aefd2d77ed20a4b353f8ebf0f4ecdd26a807d2da5aa8cd0615",
|
||||||
"https://bcr.bazel.build/modules/platforms/0.0.7/MODULE.bazel": "72fd4a0ede9ee5c021f6a8dd92b503e089f46c227ba2813ff183b71616034814",
|
"https://bcr.bazel.build/modules/platforms/0.0.7/MODULE.bazel": "72fd4a0ede9ee5c021f6a8dd92b503e089f46c227ba2813ff183b71616034814",
|
||||||
"https://bcr.bazel.build/modules/platforms/0.0.8/MODULE.bazel": "9f142c03e348f6d263719f5074b21ef3adf0b139ee4c5133e2aa35664da9eb2d",
|
"https://bcr.bazel.build/modules/platforms/0.0.8/MODULE.bazel": "9f142c03e348f6d263719f5074b21ef3adf0b139ee4c5133e2aa35664da9eb2d",
|
||||||
"https://bcr.bazel.build/modules/platforms/0.0.9/MODULE.bazel": "4a87a60c927b56ddd67db50c89acaa62f4ce2a1d2149ccb63ffd871d5ce29ebc",
|
"https://bcr.bazel.build/modules/platforms/0.0.9/MODULE.bazel": "4a87a60c927b56ddd67db50c89acaa62f4ce2a1d2149ccb63ffd871d5ce29ebc",
|
||||||
|
"https://bcr.bazel.build/modules/platforms/1.0.0/MODULE.bazel": "f05feb42b48f1b3c225e4ccf351f367be0371411a803198ec34a389fb22aa580",
|
||||||
|
"https://bcr.bazel.build/modules/platforms/1.0.0/source.json": "f4ff1fd412e0246fd38c82328eb209130ead81d62dcd5a9e40910f867f733d96",
|
||||||
"https://bcr.bazel.build/modules/protobuf/21.7/MODULE.bazel": "a5a29bb89544f9b97edce05642fac225a808b5b7be74038ea3640fae2f8e66a7",
|
"https://bcr.bazel.build/modules/protobuf/21.7/MODULE.bazel": "a5a29bb89544f9b97edce05642fac225a808b5b7be74038ea3640fae2f8e66a7",
|
||||||
"https://bcr.bazel.build/modules/protobuf/27.0/MODULE.bazel": "7873b60be88844a0a1d8f80b9d5d20cfbd8495a689b8763e76c6372998d3f64c",
|
"https://bcr.bazel.build/modules/protobuf/27.0/MODULE.bazel": "7873b60be88844a0a1d8f80b9d5d20cfbd8495a689b8763e76c6372998d3f64c",
|
||||||
"https://bcr.bazel.build/modules/protobuf/27.1/MODULE.bazel": "703a7b614728bb06647f965264967a8ef1c39e09e8f167b3ca0bb1fd80449c0d",
|
"https://bcr.bazel.build/modules/protobuf/27.1/MODULE.bazel": "703a7b614728bb06647f965264967a8ef1c39e09e8f167b3ca0bb1fd80449c0d",
|
||||||
|
|
@ -93,7 +99,9 @@
|
||||||
"https://bcr.bazel.build/modules/rules_cc/0.0.8/MODULE.bazel": "964c85c82cfeb6f3855e6a07054fdb159aced38e99a5eecf7bce9d53990afa3e",
|
"https://bcr.bazel.build/modules/rules_cc/0.0.8/MODULE.bazel": "964c85c82cfeb6f3855e6a07054fdb159aced38e99a5eecf7bce9d53990afa3e",
|
||||||
"https://bcr.bazel.build/modules/rules_cc/0.0.9/MODULE.bazel": "836e76439f354b89afe6a911a7adf59a6b2518fafb174483ad78a2a2fde7b1c5",
|
"https://bcr.bazel.build/modules/rules_cc/0.0.9/MODULE.bazel": "836e76439f354b89afe6a911a7adf59a6b2518fafb174483ad78a2a2fde7b1c5",
|
||||||
"https://bcr.bazel.build/modules/rules_cc/0.1.1/MODULE.bazel": "2f0222a6f229f0bf44cd711dc13c858dad98c62d52bd51d8fc3a764a83125513",
|
"https://bcr.bazel.build/modules/rules_cc/0.1.1/MODULE.bazel": "2f0222a6f229f0bf44cd711dc13c858dad98c62d52bd51d8fc3a764a83125513",
|
||||||
"https://bcr.bazel.build/modules/rules_cc/0.1.1/source.json": "d61627377bd7dd1da4652063e368d9366fc9a73920bfa396798ad92172cf645c",
|
"https://bcr.bazel.build/modules/rules_cc/0.2.4/MODULE.bazel": "1ff1223dfd24f3ecf8f028446d4a27608aa43c3f41e346d22838a4223980b8cc",
|
||||||
|
"https://bcr.bazel.build/modules/rules_cc/0.2.8/MODULE.bazel": "f1df20f0bf22c28192a794f29b501ee2018fa37a3862a1a2132ae2940a23a642",
|
||||||
|
"https://bcr.bazel.build/modules/rules_cc/0.2.8/source.json": "85087982aca15f31307bd52698316b28faa31bd2c3095a41f456afec0131344c",
|
||||||
"https://bcr.bazel.build/modules/rules_foreign_cc/0.9.0/MODULE.bazel": "c9e8c682bf75b0e7c704166d79b599f93b72cfca5ad7477df596947891feeef6",
|
"https://bcr.bazel.build/modules/rules_foreign_cc/0.9.0/MODULE.bazel": "c9e8c682bf75b0e7c704166d79b599f93b72cfca5ad7477df596947891feeef6",
|
||||||
"https://bcr.bazel.build/modules/rules_fuzzing/0.5.2/MODULE.bazel": "40c97d1144356f52905566c55811f13b299453a14ac7769dfba2ac38192337a8",
|
"https://bcr.bazel.build/modules/rules_fuzzing/0.5.2/MODULE.bazel": "40c97d1144356f52905566c55811f13b299453a14ac7769dfba2ac38192337a8",
|
||||||
"https://bcr.bazel.build/modules/rules_fuzzing/0.5.2/source.json": "c8b1e2c717646f1702290959a3302a178fb639d987ab61d548105019f11e527e",
|
"https://bcr.bazel.build/modules/rules_fuzzing/0.5.2/source.json": "c8b1e2c717646f1702290959a3302a178fb639d987ab61d548105019f11e527e",
|
||||||
|
|
@ -108,8 +116,8 @@
|
||||||
"https://bcr.bazel.build/modules/rules_java/7.2.0/MODULE.bazel": "06c0334c9be61e6cef2c8c84a7800cef502063269a5af25ceb100b192453d4ab",
|
"https://bcr.bazel.build/modules/rules_java/7.2.0/MODULE.bazel": "06c0334c9be61e6cef2c8c84a7800cef502063269a5af25ceb100b192453d4ab",
|
||||||
"https://bcr.bazel.build/modules/rules_java/7.3.2/MODULE.bazel": "50dece891cfdf1741ea230d001aa9c14398062f2b7c066470accace78e412bc2",
|
"https://bcr.bazel.build/modules/rules_java/7.3.2/MODULE.bazel": "50dece891cfdf1741ea230d001aa9c14398062f2b7c066470accace78e412bc2",
|
||||||
"https://bcr.bazel.build/modules/rules_java/7.6.1/MODULE.bazel": "2f14b7e8a1aa2f67ae92bc69d1ec0fa8d9f827c4e17ff5e5f02e91caa3b2d0fe",
|
"https://bcr.bazel.build/modules/rules_java/7.6.1/MODULE.bazel": "2f14b7e8a1aa2f67ae92bc69d1ec0fa8d9f827c4e17ff5e5f02e91caa3b2d0fe",
|
||||||
"https://bcr.bazel.build/modules/rules_java/8.12.0/MODULE.bazel": "8e6590b961f2defdfc2811c089c75716cb2f06c8a4edeb9a8d85eaa64ee2a761",
|
"https://bcr.bazel.build/modules/rules_java/8.14.0/MODULE.bazel": "717717ed40cc69994596a45aec6ea78135ea434b8402fb91b009b9151dd65615",
|
||||||
"https://bcr.bazel.build/modules/rules_java/8.12.0/source.json": "cbd5d55d9d38d4008a7d00bee5b5a5a4b6031fcd4a56515c9accbcd42c7be2ba",
|
"https://bcr.bazel.build/modules/rules_java/8.14.0/source.json": "8a88c4ca9e8759da53cddc88123880565c520503321e2566b4e33d0287a3d4bc",
|
||||||
"https://bcr.bazel.build/modules/rules_java/8.3.2/MODULE.bazel": "7336d5511ad5af0b8615fdc7477535a2e4e723a357b6713af439fe8cf0195017",
|
"https://bcr.bazel.build/modules/rules_java/8.3.2/MODULE.bazel": "7336d5511ad5af0b8615fdc7477535a2e4e723a357b6713af439fe8cf0195017",
|
||||||
"https://bcr.bazel.build/modules/rules_java/8.5.1/MODULE.bazel": "d8a9e38cc5228881f7055a6079f6f7821a073df3744d441978e7a43e20226939",
|
"https://bcr.bazel.build/modules/rules_java/8.5.1/MODULE.bazel": "d8a9e38cc5228881f7055a6079f6f7821a073df3744d441978e7a43e20226939",
|
||||||
"https://bcr.bazel.build/modules/rules_jvm_external/4.4.2/MODULE.bazel": "a56b85e418c83eb1839819f0b515c431010160383306d13ec21959ac412d2fe7",
|
"https://bcr.bazel.build/modules/rules_jvm_external/4.4.2/MODULE.bazel": "a56b85e418c83eb1839819f0b515c431010160383306d13ec21959ac412d2fe7",
|
||||||
|
|
@ -149,12 +157,11 @@
|
||||||
"https://bcr.bazel.build/modules/rules_python/1.3.0/MODULE.bazel": "8361d57eafb67c09b75bf4bbe6be360e1b8f4f18118ab48037f2bd50aa2ccb13",
|
"https://bcr.bazel.build/modules/rules_python/1.3.0/MODULE.bazel": "8361d57eafb67c09b75bf4bbe6be360e1b8f4f18118ab48037f2bd50aa2ccb13",
|
||||||
"https://bcr.bazel.build/modules/rules_python/1.5.1/MODULE.bazel": "acfe65880942d44a69129d4c5c3122d57baaf3edf58ae5a6bd4edea114906bf5",
|
"https://bcr.bazel.build/modules/rules_python/1.5.1/MODULE.bazel": "acfe65880942d44a69129d4c5c3122d57baaf3edf58ae5a6bd4edea114906bf5",
|
||||||
"https://bcr.bazel.build/modules/rules_python/1.5.1/source.json": "aa903e1bcbdfa1580f2b8e2d55100b7c18bc92d779ebb507fec896c75635f7bd",
|
"https://bcr.bazel.build/modules/rules_python/1.5.1/source.json": "aa903e1bcbdfa1580f2b8e2d55100b7c18bc92d779ebb507fec896c75635f7bd",
|
||||||
"https://bcr.bazel.build/modules/rules_rust/0.61.0/MODULE.bazel": "0318a95777b9114c8740f34b60d6d68f9cfef61e2f4b52424ca626213d33787b",
|
"https://bcr.bazel.build/modules/rules_rust/0.67.0/MODULE.bazel": "87c3816c4321352dcfd9e9e26b58e84efc5b21351ae3ef8fb5d0d57bde7237f5",
|
||||||
"https://bcr.bazel.build/modules/rules_rust/0.61.0/source.json": "d1bc743b5fa2e2abb35c436df7126a53dab0c3f35890ae6841592b2253786a63",
|
"https://bcr.bazel.build/modules/rules_rust/0.67.0/source.json": "a8ef4d3be30eb98e060cad9e5875a55b603195487f76e01b619b51a1df4641cc",
|
||||||
"https://bcr.bazel.build/modules/rules_shell/0.2.0/MODULE.bazel": "fda8a652ab3c7d8fee214de05e7a9916d8b28082234e8d2c0094505c5268ed3c",
|
"https://bcr.bazel.build/modules/rules_shell/0.2.0/MODULE.bazel": "fda8a652ab3c7d8fee214de05e7a9916d8b28082234e8d2c0094505c5268ed3c",
|
||||||
"https://bcr.bazel.build/modules/rules_shell/0.3.0/MODULE.bazel": "de4402cd12f4cc8fda2354fce179fdb068c0b9ca1ec2d2b17b3e21b24c1a937b",
|
"https://bcr.bazel.build/modules/rules_shell/0.6.1/MODULE.bazel": "72e76b0eea4e81611ef5452aa82b3da34caca0c8b7b5c0c9584338aa93bae26b",
|
||||||
"https://bcr.bazel.build/modules/rules_shell/0.4.0/MODULE.bazel": "0f8f11bb3cd11755f0b48c1de0bbcf62b4b34421023aa41a2fc74ef68d9584f0",
|
"https://bcr.bazel.build/modules/rules_shell/0.6.1/source.json": "20ec05cd5e592055e214b2da8ccb283c7f2a421ea0dc2acbf1aa792e11c03d0c",
|
||||||
"https://bcr.bazel.build/modules/rules_shell/0.4.0/source.json": "1d7fa7f941cd41dc2704ba5b4edc2e2230eea1cc600d80bd2b65838204c50b95",
|
|
||||||
"https://bcr.bazel.build/modules/stardoc/0.5.1/MODULE.bazel": "1a05d92974d0c122f5ccf09291442580317cdd859f07a8655f1db9a60374f9f8",
|
"https://bcr.bazel.build/modules/stardoc/0.5.1/MODULE.bazel": "1a05d92974d0c122f5ccf09291442580317cdd859f07a8655f1db9a60374f9f8",
|
||||||
"https://bcr.bazel.build/modules/stardoc/0.5.3/MODULE.bazel": "c7f6948dae6999bf0db32c1858ae345f112cacf98f174c7a8bb707e41b974f1c",
|
"https://bcr.bazel.build/modules/stardoc/0.5.3/MODULE.bazel": "c7f6948dae6999bf0db32c1858ae345f112cacf98f174c7a8bb707e41b974f1c",
|
||||||
"https://bcr.bazel.build/modules/stardoc/0.5.4/MODULE.bazel": "6569966df04610b8520957cb8e97cf2e9faac2c0309657c537ab51c16c18a2a4",
|
"https://bcr.bazel.build/modules/stardoc/0.5.4/MODULE.bazel": "6569966df04610b8520957cb8e97cf2e9faac2c0309657c537ab51c16c18a2a4",
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue