make tests actually fail when they should
This commit is contained in:
parent
8f287e810a
commit
d159b479d1
4 changed files with 116 additions and 53 deletions
|
|
@ -101,50 +101,17 @@ main() {
|
||||||
# Test 2: Podcast Reviews
|
# Test 2: Podcast Reviews
|
||||||
log_info "=== Podcast Reviews End-to-End Tests ==="
|
log_info "=== Podcast Reviews End-to-End Tests ==="
|
||||||
|
|
||||||
# Try to build podcast reviews targets, but don't fail if it times out
|
# Build podcast reviews targets - fail fast if build fails
|
||||||
log_info "Attempting to build podcast reviews targets (may skip if slow)..."
|
log_info "Building podcast reviews targets"
|
||||||
build_success=true
|
build_targets "$SCRIPT_DIR/examples/podcast_reviews" \
|
||||||
if ! (cd "$SCRIPT_DIR/examples/podcast_reviews" && \
|
"//:podcast_reviews_graph.build" \
|
||||||
bazel build "//:podcast_reviews_graph.build" "//:podcast_reviews_graph.service" 2>/dev/null); then
|
"//:podcast_reviews_graph.service"
|
||||||
build_success=false
|
|
||||||
fi
|
|
||||||
|
|
||||||
if [[ "$build_success" == "false" ]]; then
|
# Run podcast reviews tests - fail fast if build/test fails
|
||||||
log_warn "Podcast reviews build failed or timed out, checking for existing binaries..."
|
run_test "Podcast Reviews Simple Test" \
|
||||||
if [[ -f "$SCRIPT_DIR/examples/podcast_reviews/bazel-bin/podcast_reviews_graph.build" ]]; then
|
"$TESTS_DIR/podcast_simple_test.sh" \
|
||||||
log_info "Found existing podcast reviews binary, using it for testing"
|
"$SCRIPT_DIR/examples/podcast_reviews/bazel-bin/podcast_reviews_graph.build" \
|
||||||
else
|
"$SCRIPT_DIR/examples/podcast_reviews/bazel-bin/podcast_reviews_graph.service"
|
||||||
log_warn "Skipping podcast reviews test - no binary available"
|
|
||||||
log_info "You can manually test with: cd examples/podcast_reviews && bazel build //:podcast_reviews_graph.build"
|
|
||||||
fi
|
|
||||||
else
|
|
||||||
test_pass "Built podcast reviews targets"
|
|
||||||
fi
|
|
||||||
|
|
||||||
# Test with existing binary
|
|
||||||
if [[ -f "$SCRIPT_DIR/examples/podcast_reviews/bazel-bin/podcast_reviews_graph.build" ]]; then
|
|
||||||
log_info "Running Podcast Reviews CLI test from correct directory"
|
|
||||||
if ! (cd "$SCRIPT_DIR/examples/podcast_reviews" && \
|
|
||||||
export DATABUILD_BUILD_EVENT_LOG="sqlite:///tmp/podcast_e2e_test.db" && \
|
|
||||||
rm -f /tmp/podcast_e2e_test.db && \
|
|
||||||
bazel-bin/podcast_reviews_graph.build "reviews/date=2020-01-01" > /tmp/podcast_e2e_output.log 2>&1); then
|
|
||||||
log_error "Podcast Reviews CLI test failed:"
|
|
||||||
cat /tmp/podcast_e2e_output.log
|
|
||||||
log_warn "Podcast reviews test failed, but continuing..."
|
|
||||||
else
|
|
||||||
# Check that events were generated
|
|
||||||
if [[ -f /tmp/podcast_e2e_test.db ]]; then
|
|
||||||
local events=$(sqlite3 /tmp/podcast_e2e_test.db "SELECT COUNT(*) FROM build_events;" 2>/dev/null || echo "0")
|
|
||||||
if [[ "$events" -gt 0 ]]; then
|
|
||||||
test_pass "Podcast Reviews CLI test - generated $events events"
|
|
||||||
else
|
|
||||||
log_warn "Podcast Reviews CLI test - no events generated"
|
|
||||||
fi
|
|
||||||
else
|
|
||||||
log_warn "Podcast Reviews CLI test - no database created"
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
|
|
||||||
# Test 3: Core DataBuild Tests (if any exist)
|
# Test 3: Core DataBuild Tests (if any exist)
|
||||||
log_info "=== Core DataBuild Tests ==="
|
log_info "=== Core DataBuild Tests ==="
|
||||||
|
|
|
||||||
|
|
@ -289,11 +289,43 @@ test_event_comparison() {
|
||||||
return 1
|
return 1
|
||||||
fi
|
fi
|
||||||
|
|
||||||
# Events should be similar in count (allowing for some variation)
|
# Detailed event count validation (matching simple_test.sh approach)
|
||||||
if [[ $((cli_event_count - service_event_count)) -gt 3 ]] || [[ $((service_event_count - cli_event_count)) -gt 3 ]]; then
|
log_info "Performing detailed event count validation..."
|
||||||
log_warn "Event counts differ significantly: CLI=$cli_event_count, Service=$service_event_count"
|
|
||||||
|
local cli_total_events=$(count_build_events "$cli_db_path")
|
||||||
|
local service_total_events=$(count_build_events "$service_db_path")
|
||||||
|
|
||||||
|
log_info "Total events: CLI=$cli_total_events, Service=$service_total_events"
|
||||||
|
|
||||||
|
# Count events by type using the same approach as simple_test.sh
|
||||||
|
local cli_job_events=$(sqlite3 "$cli_db_path" "SELECT COUNT(*) FROM build_events WHERE event_type = 'job';" 2>/dev/null || echo "0")
|
||||||
|
local cli_partition_events=$(sqlite3 "$cli_db_path" "SELECT COUNT(*) FROM build_events WHERE event_type = 'partition';" 2>/dev/null || echo "0")
|
||||||
|
local cli_request_events=$(sqlite3 "$cli_db_path" "SELECT COUNT(*) FROM build_events WHERE event_type = 'build_request';" 2>/dev/null || echo "0")
|
||||||
|
|
||||||
|
local service_job_events=$(sqlite3 "$service_db_path" "SELECT COUNT(*) FROM build_events WHERE event_type = 'job';" 2>/dev/null || echo "0")
|
||||||
|
local service_partition_events=$(sqlite3 "$service_db_path" "SELECT COUNT(*) FROM build_events WHERE event_type = 'partition';" 2>/dev/null || echo "0")
|
||||||
|
local service_request_events=$(sqlite3 "$service_db_path" "SELECT COUNT(*) FROM build_events WHERE event_type = 'build_request';" 2>/dev/null || echo "0")
|
||||||
|
|
||||||
|
log_info "Event breakdown:"
|
||||||
|
log_info " Job events: CLI=$cli_job_events, Service=$service_job_events"
|
||||||
|
log_info " Partition events: CLI=$cli_partition_events, Service=$service_partition_events"
|
||||||
|
log_info " Request events: CLI=$cli_request_events, Service=$service_request_events"
|
||||||
|
|
||||||
|
# Validate core events are identical (job, partition, and request events should all match now)
|
||||||
|
if [[ "$cli_job_events" -eq "$service_job_events" ]] && [[ "$cli_partition_events" -eq "$service_partition_events" ]] && [[ "$cli_request_events" -eq "$service_request_events" ]]; then
|
||||||
|
log_info "✅ All build events (job, partition, and request) are identical"
|
||||||
else
|
else
|
||||||
log_info "Event counts are similar: CLI=$cli_event_count, Service=$service_event_count"
|
log_error "❌ Build events differ between CLI and Service - this indicates a problem"
|
||||||
|
log_error "Expected CLI and Service to emit identical event counts after alignment"
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Validate total event counts are identical
|
||||||
|
if [[ "$cli_total_events" -eq "$service_total_events" ]]; then
|
||||||
|
log_info "✅ Total event counts are identical: $cli_total_events events each"
|
||||||
|
else
|
||||||
|
log_error "❌ Total event counts differ: CLI=$cli_total_events, Service=$service_total_events"
|
||||||
|
return 1
|
||||||
fi
|
fi
|
||||||
|
|
||||||
test_pass "Event comparison test"
|
test_pass "Event comparison test"
|
||||||
|
|
|
||||||
|
|
@ -355,12 +355,43 @@ test_consistency_validation() {
|
||||||
log_info "Event counts are consistent: CLI=$cli_event_count, Service=$service_event_count"
|
log_info "Event counts are consistent: CLI=$cli_event_count, Service=$service_event_count"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
# Check event types distribution
|
# Detailed event count validation (matching simple_test.sh approach)
|
||||||
local cli_completed=$(jq '[.[] | select(.event_type == "COMPLETED")] | length' "$cli_events_file")
|
log_info "Performing detailed event count validation..."
|
||||||
local service_completed=$(jq '[.[] | select(.event_type == "COMPLETED")] | length' "$service_events_file")
|
|
||||||
|
|
||||||
if [[ "$cli_completed" -ne "$service_completed" ]]; then
|
local cli_total_events=$(count_build_events "$cli_db_consistency")
|
||||||
log_warn "Completed event counts differ: CLI=$cli_completed, Service=$service_completed"
|
local service_total_events=$(count_build_events "$service_db_consistency")
|
||||||
|
|
||||||
|
log_info "Total events: CLI=$cli_total_events, Service=$service_total_events"
|
||||||
|
|
||||||
|
# Count events by type using the same approach as simple_test.sh
|
||||||
|
local cli_job_events=$(sqlite3 "$cli_db_consistency" "SELECT COUNT(*) FROM build_events WHERE event_type = 'job';" 2>/dev/null || echo "0")
|
||||||
|
local cli_partition_events=$(sqlite3 "$cli_db_consistency" "SELECT COUNT(*) FROM build_events WHERE event_type = 'partition';" 2>/dev/null || echo "0")
|
||||||
|
local cli_request_events=$(sqlite3 "$cli_db_consistency" "SELECT COUNT(*) FROM build_events WHERE event_type = 'build_request';" 2>/dev/null || echo "0")
|
||||||
|
|
||||||
|
local service_job_events=$(sqlite3 "$service_db_consistency" "SELECT COUNT(*) FROM build_events WHERE event_type = 'job';" 2>/dev/null || echo "0")
|
||||||
|
local service_partition_events=$(sqlite3 "$service_db_consistency" "SELECT COUNT(*) FROM build_events WHERE event_type = 'partition';" 2>/dev/null || echo "0")
|
||||||
|
local service_request_events=$(sqlite3 "$service_db_consistency" "SELECT COUNT(*) FROM build_events WHERE event_type = 'build_request';" 2>/dev/null || echo "0")
|
||||||
|
|
||||||
|
log_info "Event breakdown:"
|
||||||
|
log_info " Job events: CLI=$cli_job_events, Service=$service_job_events"
|
||||||
|
log_info " Partition events: CLI=$cli_partition_events, Service=$service_partition_events"
|
||||||
|
log_info " Request events: CLI=$cli_request_events, Service=$service_request_events"
|
||||||
|
|
||||||
|
# Validate core events are identical (job, partition, and request events should all match now)
|
||||||
|
if [[ "$cli_job_events" -eq "$service_job_events" ]] && [[ "$cli_partition_events" -eq "$service_partition_events" ]] && [[ "$cli_request_events" -eq "$service_request_events" ]]; then
|
||||||
|
log_info "✅ All build events (job, partition, and request) are identical"
|
||||||
|
else
|
||||||
|
log_error "❌ Build events differ between CLI and Service - this indicates a problem"
|
||||||
|
log_error "Expected CLI and Service to emit identical event counts after alignment"
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Validate total event counts are identical
|
||||||
|
if [[ "$cli_total_events" -eq "$service_total_events" ]]; then
|
||||||
|
log_info "✅ Total event counts are identical: $cli_total_events events each"
|
||||||
|
else
|
||||||
|
log_error "❌ Total event counts differ: CLI=$cli_total_events, Service=$service_total_events"
|
||||||
|
return 1
|
||||||
fi
|
fi
|
||||||
|
|
||||||
test_pass "Consistency validation test"
|
test_pass "Consistency validation test"
|
||||||
|
|
|
||||||
|
|
@ -115,10 +115,43 @@ else
|
||||||
exit 1
|
exit 1
|
||||||
fi
|
fi
|
||||||
|
|
||||||
# Compare event counts (should be similar)
|
# Compare event counts with detailed analysis
|
||||||
if [[ "$CLI_EVENTS" -gt 0 ]] && [[ "$SERVICE_EVENTS" -gt 0 ]]; then
|
if [[ "$CLI_EVENTS" -gt 0 ]] && [[ "$SERVICE_EVENTS" -gt 0 ]]; then
|
||||||
echo "[INFO] Both CLI and Service generated events successfully"
|
echo "[INFO] Both CLI and Service generated events successfully"
|
||||||
echo "[INFO] CLI: $CLI_EVENTS events, Service: $SERVICE_EVENTS events"
|
echo "[INFO] CLI: $CLI_EVENTS events, Service: $SERVICE_EVENTS events"
|
||||||
|
|
||||||
|
# Detailed event comparison
|
||||||
|
echo "[INFO] Analyzing event distribution..."
|
||||||
|
|
||||||
|
CLI_JOB_EVENTS=$(sqlite3 /tmp/podcast_test_cli.db "SELECT COUNT(*) FROM build_events WHERE event_type = 'job';" 2>/dev/null || echo "0")
|
||||||
|
CLI_PARTITION_EVENTS=$(sqlite3 /tmp/podcast_test_cli.db "SELECT COUNT(*) FROM build_events WHERE event_type = 'partition';" 2>/dev/null || echo "0")
|
||||||
|
CLI_REQUEST_EVENTS=$(sqlite3 /tmp/podcast_test_cli.db "SELECT COUNT(*) FROM build_events WHERE event_type = 'build_request';" 2>/dev/null || echo "0")
|
||||||
|
|
||||||
|
SERVICE_JOB_EVENTS=$(sqlite3 "$SERVICE_DB_PATH" "SELECT COUNT(*) FROM build_events WHERE event_type = 'job';" 2>/dev/null || echo "0")
|
||||||
|
SERVICE_PARTITION_EVENTS=$(sqlite3 "$SERVICE_DB_PATH" "SELECT COUNT(*) FROM build_events WHERE event_type = 'partition';" 2>/dev/null || echo "0")
|
||||||
|
SERVICE_REQUEST_EVENTS=$(sqlite3 "$SERVICE_DB_PATH" "SELECT COUNT(*) FROM build_events WHERE event_type = 'build_request';" 2>/dev/null || echo "0")
|
||||||
|
|
||||||
|
echo "[INFO] Event breakdown:"
|
||||||
|
echo "[INFO] Job events: CLI=$CLI_JOB_EVENTS, Service=$SERVICE_JOB_EVENTS"
|
||||||
|
echo "[INFO] Partition events: CLI=$CLI_PARTITION_EVENTS, Service=$SERVICE_PARTITION_EVENTS"
|
||||||
|
echo "[INFO] Request events: CLI=$CLI_REQUEST_EVENTS, Service=$SERVICE_REQUEST_EVENTS"
|
||||||
|
|
||||||
|
# Validate core events are identical (job, partition, and request events should all match now)
|
||||||
|
if [[ "$CLI_JOB_EVENTS" -eq "$SERVICE_JOB_EVENTS" ]] && [[ "$CLI_PARTITION_EVENTS" -eq "$SERVICE_PARTITION_EVENTS" ]] && [[ "$CLI_REQUEST_EVENTS" -eq "$SERVICE_REQUEST_EVENTS" ]]; then
|
||||||
|
echo "[INFO] ✅ All build events (job, partition, and request) are identical"
|
||||||
|
else
|
||||||
|
echo "[ERROR] ❌ Build events differ between CLI and Service - this indicates a problem"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Validate total event counts are identical
|
||||||
|
if [[ "$CLI_EVENTS" -eq "$SERVICE_EVENTS" ]]; then
|
||||||
|
echo "[INFO] ✅ Total event counts are identical: $CLI_EVENTS events each"
|
||||||
|
else
|
||||||
|
echo "[ERROR] ❌ Total event counts differ: CLI=$CLI_EVENTS, Service=$SERVICE_EVENTS"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
else
|
else
|
||||||
echo "[ERROR] One or both builds generated no events"
|
echo "[ERROR] One or both builds generated no events"
|
||||||
exit 1
|
exit 1
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue