stuart/databuild
1
1
Fork 0
Code Issues 1 Pull requests Projects Releases Packages Wiki Activity Actions

Make event emission match between cli and service build paths

Browse source
This commit is contained in:
Stuart Axelbrooke 2025-07-07 22:09:11 -07:00
parent 064a31606a
commit 8f287e810a
5 changed files with 160 additions and 4 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

38
databuild/graph/analyze.rs
View file

@ -618,6 +618,44 @@ async fn main() {
None
};
// Check if this is a CLI build (not service-initiated)
let is_cli_mode = env::var("DATABUILD_CLI_MODE").is_ok();
// Emit orchestration events for CLI mode to match Service behavior
if is_cli_mode {
if let Some(ref event_log) = build_event_log {
// Emit "Build request received" event
let event = create_build_event(
build_request_id.clone(),
crate::build_event::EventType::BuildRequestEvent(BuildRequestEvent {
status: BuildRequestStatus::BuildRequestReceived as i32,
requested_partitions: args.iter()
.map(|p| PartitionRef { str: p.clone() })
.collect(),
message: "Build request received".to_string(),
})
);
if let Err(e) = event_log.append_event(event).await {
error!("Failed to log build request received event: {}", e);
}
// Emit "Starting build planning" event
let event = create_build_event(
build_request_id.clone(),
crate::build_event::EventType::BuildRequestEvent(BuildRequestEvent {
status: BuildRequestStatus::BuildRequestPlanning as i32,
requested_partitions: args.iter()
.map(|p| PartitionRef { str: p.clone() })
.collect(),
message: "Starting build planning".to_string(),
})
);
if let Err(e) = event_log.append_event(event).await {
error!("Failed to log build planning event: {}", e);
}
}
}
match mode.as_str() {
"plan" => {
// Get output refs from command line arguments

54
databuild/graph/execute.rs
View file

@ -367,7 +367,28 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
info!("Executing job graph with {} nodes", graph.nodes.len());
// Log build request execution start
// Check if this is a CLI build (not service-initiated)
let is_cli_mode = std::env::var("DATABUILD_CLI_MODE").is_ok();
// Emit orchestration events for CLI mode to match Service behavior
if is_cli_mode {
if let Some(ref event_log) = build_event_log {
// Emit "Starting build execution" event (matches Service pattern)
let event = create_build_event(
build_request_id.clone(),
EventType::BuildRequestEvent(BuildRequestEvent {
status: BuildRequestStatus::BuildRequestExecuting as i32,
requested_partitions: graph.outputs.clone(),
message: "Starting build execution".to_string(),
})
);
if let Err(e) = event_log.append_event(event).await {
error!("Failed to log build execution event: {}", e);
}
}
}
// Log build request execution start (existing detailed event)
if let Some(ref event_log) = build_event_log {
let event = create_build_event(
build_request_id.clone(),
@ -648,7 +669,36 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
info!("Execution complete: {} succeeded, {} failed", success_count, failure_count);
// Log final build request status
// Emit orchestration completion events for CLI mode to match Service behavior
if is_cli_mode {
if let Some(ref event_log) = build_event_log {
let orchestration_message = if failure_count > 0 || fail_fast_triggered {
"Build request failed"
} else {
"Build request completed successfully"
};
let orchestration_status = if failure_count > 0 || fail_fast_triggered {
BuildRequestStatus::BuildRequestFailed
} else {
BuildRequestStatus::BuildRequestCompleted
};
let event = create_build_event(
build_request_id.clone(),
EventType::BuildRequestEvent(BuildRequestEvent {
status: orchestration_status as i32,
requested_partitions: graph.outputs.clone(),
message: orchestration_message.to_string(),
})
);
if let Err(e) = event_log.append_event(event).await {
error!("Failed to log build request completion event: {}", e);
}
}
}
// Log final build request status (existing detailed event)
if let Some(ref event_log) = build_event_log {
let final_status = if failure_count > 0 || fail_fast_triggered {
BuildRequestStatus::BuildRequestFailed

5
databuild/rules.bzl
View file

@ -635,7 +635,10 @@ def _databuild_graph_build_impl(ctx):
output = script,
is_executable = True,
content = "#!/bin/bash\n\n" + RUNFILES_PREFIX + """
# TODO these rlocation commands resolve to nothing - does it expect to be
# Set CLI mode to indicate this is a direct CLI build (not service-initiated)
export DATABUILD_CLI_MODE=true
# Run analysis and execution phases
$(rlocation _main/{analyze_path}) $@ | $(rlocation _main/{exec_path})
""".format(
analyze_path = ctx.attr.analyze.files_to_run.executable.short_path,

40
tests/end_to_end/lib/db_utils.sh
View file

@ -69,6 +69,46 @@ compare_build_events() {
fi
}
# Compare core build events (job and partition only, ignoring request orchestration)
compare_core_build_events() {
local db1="$1"
local db2="$2"
log_info "Comparing core build events (job and partition) between databases" >&2
# Count core events by type
local db1_job=$(count_build_events "$db1" "job")
local db1_partition=$(count_build_events "$db1" "partition")
local db2_job=$(count_build_events "$db2" "job")
local db2_partition=$(count_build_events "$db2" "partition")
log_info "Database 1: $db1_job job events, $db1_partition partition events" >&2
log_info "Database 2: $db2_job job events, $db2_partition partition events" >&2
# Check if core events match
if [[ "$db1_job" -eq "$db2_job" ]] && [[ "$db1_partition" -eq "$db2_partition" ]]; then
log_info "Core build events match" >&2
return 0
else
log_error "Core build events differ" >&2
return 1
fi
}
# Get detailed event breakdown
get_event_breakdown() {
local db_path="$1"
if [[ ! -f "$db_path" ]]; then
echo "Database not found: $db_path" >&2
return 1
fi
echo "Event breakdown for $db_path:" >&2
sqlite3 "$db_path" \
"SELECT event_type, COUNT(*) FROM build_events GROUP BY event_type ORDER BY event_type;" >&2
}
# Count events in database
count_build_events() {
local db_path="$1"

27
tests/end_to_end/simple_test.sh
View file

@ -117,10 +117,35 @@ else
exit 1
fi
# Compare event counts (should be similar)
# Compare event counts with detailed analysis
if [[ "$CLI_EVENTS" -gt 0 ]] && [[ "$SERVICE_EVENTS" -gt 0 ]]; then
echo "[INFO] Both CLI and Service generated events successfully"
echo "[INFO] CLI: $CLI_EVENTS events, Service: $SERVICE_EVENTS events"
# Detailed event comparison
echo "[INFO] Analyzing event distribution..."
CLI_JOB_EVENTS=$(sqlite3 /tmp/simple_test_cli.db "SELECT COUNT(*) FROM build_events WHERE event_type = 'job';" 2>/dev/null || echo "0")
CLI_PARTITION_EVENTS=$(sqlite3 /tmp/simple_test_cli.db "SELECT COUNT(*) FROM build_events WHERE event_type = 'partition';" 2>/dev/null || echo "0")
CLI_REQUEST_EVENTS=$(sqlite3 /tmp/simple_test_cli.db "SELECT COUNT(*) FROM build_events WHERE event_type = 'build_request';" 2>/dev/null || echo "0")
SERVICE_JOB_EVENTS=$(sqlite3 "$SERVICE_DB_PATH" "SELECT COUNT(*) FROM build_events WHERE event_type = 'job';" 2>/dev/null || echo "0")
SERVICE_PARTITION_EVENTS=$(sqlite3 "$SERVICE_DB_PATH" "SELECT COUNT(*) FROM build_events WHERE event_type = 'partition';" 2>/dev/null || echo "0")
SERVICE_REQUEST_EVENTS=$(sqlite3 "$SERVICE_DB_PATH" "SELECT COUNT(*) FROM build_events WHERE event_type = 'build_request';" 2>/dev/null || echo "0")
echo "[INFO] Event breakdown:"
echo "[INFO] Job events: CLI=$CLI_JOB_EVENTS, Service=$SERVICE_JOB_EVENTS"
echo "[INFO] Partition events: CLI=$CLI_PARTITION_EVENTS, Service=$SERVICE_PARTITION_EVENTS"
echo "[INFO] Request events: CLI=$CLI_REQUEST_EVENTS, Service=$SERVICE_REQUEST_EVENTS"
# Validate core events are identical
if [[ "$CLI_JOB_EVENTS" -eq "$SERVICE_JOB_EVENTS" ]] && [[ "$CLI_PARTITION_EVENTS" -eq "$SERVICE_PARTITION_EVENTS" ]] && [[ "$CLI_REQUEST_EVENTS" -eq "$SERVICE_REQUEST_EVENTS" ]]; then
echo "[INFO] ✅ Core build events (job and partition) are identical"
else
echo "[ERROR] ❌ Core build events differ - this indicates a problem"
exit 1
fi
else
echo "[ERROR] One or both builds generated no events"
exit 1