From d159b479d18bfcd36b3bfce5e444ccfed5244523 Mon Sep 17 00:00:00 2001 From: Stuart Axelbrooke Date: Mon, 7 Jul 2025 22:35:42 -0700 Subject: [PATCH] make tests actually fail when they should --- run_e2e_tests.sh | 53 +++++------------------- tests/end_to_end/basic_graph_test.sh | 40 ++++++++++++++++-- tests/end_to_end/podcast_reviews_test.sh | 41 +++++++++++++++--- tests/end_to_end/podcast_simple_test.sh | 35 +++++++++++++++- 4 files changed, 116 insertions(+), 53 deletions(-) diff --git a/run_e2e_tests.sh b/run_e2e_tests.sh index c3e0c61..51ce773 100755 --- a/run_e2e_tests.sh +++ b/run_e2e_tests.sh @@ -101,50 +101,17 @@ main() { # Test 2: Podcast Reviews log_info "=== Podcast Reviews End-to-End Tests ===" - # Try to build podcast reviews targets, but don't fail if it times out - log_info "Attempting to build podcast reviews targets (may skip if slow)..." - build_success=true - if ! (cd "$SCRIPT_DIR/examples/podcast_reviews" && \ - bazel build "//:podcast_reviews_graph.build" "//:podcast_reviews_graph.service" 2>/dev/null); then - build_success=false - fi + # Build podcast reviews targets - fail fast if build fails + log_info "Building podcast reviews targets" + build_targets "$SCRIPT_DIR/examples/podcast_reviews" \ + "//:podcast_reviews_graph.build" \ + "//:podcast_reviews_graph.service" - if [[ "$build_success" == "false" ]]; then - log_warn "Podcast reviews build failed or timed out, checking for existing binaries..." - if [[ -f "$SCRIPT_DIR/examples/podcast_reviews/bazel-bin/podcast_reviews_graph.build" ]]; then - log_info "Found existing podcast reviews binary, using it for testing" - else - log_warn "Skipping podcast reviews test - no binary available" - log_info "You can manually test with: cd examples/podcast_reviews && bazel build //:podcast_reviews_graph.build" - fi - else - test_pass "Built podcast reviews targets" - fi - - # Test with existing binary - if [[ -f "$SCRIPT_DIR/examples/podcast_reviews/bazel-bin/podcast_reviews_graph.build" ]]; then - log_info "Running Podcast Reviews CLI test from correct directory" - if ! (cd "$SCRIPT_DIR/examples/podcast_reviews" && \ - export DATABUILD_BUILD_EVENT_LOG="sqlite:///tmp/podcast_e2e_test.db" && \ - rm -f /tmp/podcast_e2e_test.db && \ - bazel-bin/podcast_reviews_graph.build "reviews/date=2020-01-01" > /tmp/podcast_e2e_output.log 2>&1); then - log_error "Podcast Reviews CLI test failed:" - cat /tmp/podcast_e2e_output.log - log_warn "Podcast reviews test failed, but continuing..." - else - # Check that events were generated - if [[ -f /tmp/podcast_e2e_test.db ]]; then - local events=$(sqlite3 /tmp/podcast_e2e_test.db "SELECT COUNT(*) FROM build_events;" 2>/dev/null || echo "0") - if [[ "$events" -gt 0 ]]; then - test_pass "Podcast Reviews CLI test - generated $events events" - else - log_warn "Podcast Reviews CLI test - no events generated" - fi - else - log_warn "Podcast Reviews CLI test - no database created" - fi - fi - fi + # Run podcast reviews tests - fail fast if build/test fails + run_test "Podcast Reviews Simple Test" \ + "$TESTS_DIR/podcast_simple_test.sh" \ + "$SCRIPT_DIR/examples/podcast_reviews/bazel-bin/podcast_reviews_graph.build" \ + "$SCRIPT_DIR/examples/podcast_reviews/bazel-bin/podcast_reviews_graph.service" # Test 3: Core DataBuild Tests (if any exist) log_info "=== Core DataBuild Tests ===" diff --git a/tests/end_to_end/basic_graph_test.sh b/tests/end_to_end/basic_graph_test.sh index 830fef1..edaca6a 100755 --- a/tests/end_to_end/basic_graph_test.sh +++ b/tests/end_to_end/basic_graph_test.sh @@ -289,11 +289,43 @@ test_event_comparison() { return 1 fi - # Events should be similar in count (allowing for some variation) - if [[ $((cli_event_count - service_event_count)) -gt 3 ]] || [[ $((service_event_count - cli_event_count)) -gt 3 ]]; then - log_warn "Event counts differ significantly: CLI=$cli_event_count, Service=$service_event_count" + # Detailed event count validation (matching simple_test.sh approach) + log_info "Performing detailed event count validation..." + + local cli_total_events=$(count_build_events "$cli_db_path") + local service_total_events=$(count_build_events "$service_db_path") + + log_info "Total events: CLI=$cli_total_events, Service=$service_total_events" + + # Count events by type using the same approach as simple_test.sh + local cli_job_events=$(sqlite3 "$cli_db_path" "SELECT COUNT(*) FROM build_events WHERE event_type = 'job';" 2>/dev/null || echo "0") + local cli_partition_events=$(sqlite3 "$cli_db_path" "SELECT COUNT(*) FROM build_events WHERE event_type = 'partition';" 2>/dev/null || echo "0") + local cli_request_events=$(sqlite3 "$cli_db_path" "SELECT COUNT(*) FROM build_events WHERE event_type = 'build_request';" 2>/dev/null || echo "0") + + local service_job_events=$(sqlite3 "$service_db_path" "SELECT COUNT(*) FROM build_events WHERE event_type = 'job';" 2>/dev/null || echo "0") + local service_partition_events=$(sqlite3 "$service_db_path" "SELECT COUNT(*) FROM build_events WHERE event_type = 'partition';" 2>/dev/null || echo "0") + local service_request_events=$(sqlite3 "$service_db_path" "SELECT COUNT(*) FROM build_events WHERE event_type = 'build_request';" 2>/dev/null || echo "0") + + log_info "Event breakdown:" + log_info " Job events: CLI=$cli_job_events, Service=$service_job_events" + log_info " Partition events: CLI=$cli_partition_events, Service=$service_partition_events" + log_info " Request events: CLI=$cli_request_events, Service=$service_request_events" + + # Validate core events are identical (job, partition, and request events should all match now) + if [[ "$cli_job_events" -eq "$service_job_events" ]] && [[ "$cli_partition_events" -eq "$service_partition_events" ]] && [[ "$cli_request_events" -eq "$service_request_events" ]]; then + log_info "✅ All build events (job, partition, and request) are identical" else - log_info "Event counts are similar: CLI=$cli_event_count, Service=$service_event_count" + log_error "❌ Build events differ between CLI and Service - this indicates a problem" + log_error "Expected CLI and Service to emit identical event counts after alignment" + return 1 + fi + + # Validate total event counts are identical + if [[ "$cli_total_events" -eq "$service_total_events" ]]; then + log_info "✅ Total event counts are identical: $cli_total_events events each" + else + log_error "❌ Total event counts differ: CLI=$cli_total_events, Service=$service_total_events" + return 1 fi test_pass "Event comparison test" diff --git a/tests/end_to_end/podcast_reviews_test.sh b/tests/end_to_end/podcast_reviews_test.sh index f18abf5..3622f55 100755 --- a/tests/end_to_end/podcast_reviews_test.sh +++ b/tests/end_to_end/podcast_reviews_test.sh @@ -355,12 +355,43 @@ test_consistency_validation() { log_info "Event counts are consistent: CLI=$cli_event_count, Service=$service_event_count" fi - # Check event types distribution - local cli_completed=$(jq '[.[] | select(.event_type == "COMPLETED")] | length' "$cli_events_file") - local service_completed=$(jq '[.[] | select(.event_type == "COMPLETED")] | length' "$service_events_file") + # Detailed event count validation (matching simple_test.sh approach) + log_info "Performing detailed event count validation..." - if [[ "$cli_completed" -ne "$service_completed" ]]; then - log_warn "Completed event counts differ: CLI=$cli_completed, Service=$service_completed" + local cli_total_events=$(count_build_events "$cli_db_consistency") + local service_total_events=$(count_build_events "$service_db_consistency") + + log_info "Total events: CLI=$cli_total_events, Service=$service_total_events" + + # Count events by type using the same approach as simple_test.sh + local cli_job_events=$(sqlite3 "$cli_db_consistency" "SELECT COUNT(*) FROM build_events WHERE event_type = 'job';" 2>/dev/null || echo "0") + local cli_partition_events=$(sqlite3 "$cli_db_consistency" "SELECT COUNT(*) FROM build_events WHERE event_type = 'partition';" 2>/dev/null || echo "0") + local cli_request_events=$(sqlite3 "$cli_db_consistency" "SELECT COUNT(*) FROM build_events WHERE event_type = 'build_request';" 2>/dev/null || echo "0") + + local service_job_events=$(sqlite3 "$service_db_consistency" "SELECT COUNT(*) FROM build_events WHERE event_type = 'job';" 2>/dev/null || echo "0") + local service_partition_events=$(sqlite3 "$service_db_consistency" "SELECT COUNT(*) FROM build_events WHERE event_type = 'partition';" 2>/dev/null || echo "0") + local service_request_events=$(sqlite3 "$service_db_consistency" "SELECT COUNT(*) FROM build_events WHERE event_type = 'build_request';" 2>/dev/null || echo "0") + + log_info "Event breakdown:" + log_info " Job events: CLI=$cli_job_events, Service=$service_job_events" + log_info " Partition events: CLI=$cli_partition_events, Service=$service_partition_events" + log_info " Request events: CLI=$cli_request_events, Service=$service_request_events" + + # Validate core events are identical (job, partition, and request events should all match now) + if [[ "$cli_job_events" -eq "$service_job_events" ]] && [[ "$cli_partition_events" -eq "$service_partition_events" ]] && [[ "$cli_request_events" -eq "$service_request_events" ]]; then + log_info "✅ All build events (job, partition, and request) are identical" + else + log_error "❌ Build events differ between CLI and Service - this indicates a problem" + log_error "Expected CLI and Service to emit identical event counts after alignment" + return 1 + fi + + # Validate total event counts are identical + if [[ "$cli_total_events" -eq "$service_total_events" ]]; then + log_info "✅ Total event counts are identical: $cli_total_events events each" + else + log_error "❌ Total event counts differ: CLI=$cli_total_events, Service=$service_total_events" + return 1 fi test_pass "Consistency validation test" diff --git a/tests/end_to_end/podcast_simple_test.sh b/tests/end_to_end/podcast_simple_test.sh index 0b9590b..2b7df01 100755 --- a/tests/end_to_end/podcast_simple_test.sh +++ b/tests/end_to_end/podcast_simple_test.sh @@ -115,10 +115,43 @@ else exit 1 fi -# Compare event counts (should be similar) +# Compare event counts with detailed analysis if [[ "$CLI_EVENTS" -gt 0 ]] && [[ "$SERVICE_EVENTS" -gt 0 ]]; then echo "[INFO] Both CLI and Service generated events successfully" echo "[INFO] CLI: $CLI_EVENTS events, Service: $SERVICE_EVENTS events" + + # Detailed event comparison + echo "[INFO] Analyzing event distribution..." + + CLI_JOB_EVENTS=$(sqlite3 /tmp/podcast_test_cli.db "SELECT COUNT(*) FROM build_events WHERE event_type = 'job';" 2>/dev/null || echo "0") + CLI_PARTITION_EVENTS=$(sqlite3 /tmp/podcast_test_cli.db "SELECT COUNT(*) FROM build_events WHERE event_type = 'partition';" 2>/dev/null || echo "0") + CLI_REQUEST_EVENTS=$(sqlite3 /tmp/podcast_test_cli.db "SELECT COUNT(*) FROM build_events WHERE event_type = 'build_request';" 2>/dev/null || echo "0") + + SERVICE_JOB_EVENTS=$(sqlite3 "$SERVICE_DB_PATH" "SELECT COUNT(*) FROM build_events WHERE event_type = 'job';" 2>/dev/null || echo "0") + SERVICE_PARTITION_EVENTS=$(sqlite3 "$SERVICE_DB_PATH" "SELECT COUNT(*) FROM build_events WHERE event_type = 'partition';" 2>/dev/null || echo "0") + SERVICE_REQUEST_EVENTS=$(sqlite3 "$SERVICE_DB_PATH" "SELECT COUNT(*) FROM build_events WHERE event_type = 'build_request';" 2>/dev/null || echo "0") + + echo "[INFO] Event breakdown:" + echo "[INFO] Job events: CLI=$CLI_JOB_EVENTS, Service=$SERVICE_JOB_EVENTS" + echo "[INFO] Partition events: CLI=$CLI_PARTITION_EVENTS, Service=$SERVICE_PARTITION_EVENTS" + echo "[INFO] Request events: CLI=$CLI_REQUEST_EVENTS, Service=$SERVICE_REQUEST_EVENTS" + + # Validate core events are identical (job, partition, and request events should all match now) + if [[ "$CLI_JOB_EVENTS" -eq "$SERVICE_JOB_EVENTS" ]] && [[ "$CLI_PARTITION_EVENTS" -eq "$SERVICE_PARTITION_EVENTS" ]] && [[ "$CLI_REQUEST_EVENTS" -eq "$SERVICE_REQUEST_EVENTS" ]]; then + echo "[INFO] ✅ All build events (job, partition, and request) are identical" + else + echo "[ERROR] ❌ Build events differ between CLI and Service - this indicates a problem" + exit 1 + fi + + # Validate total event counts are identical + if [[ "$CLI_EVENTS" -eq "$SERVICE_EVENTS" ]]; then + echo "[INFO] ✅ Total event counts are identical: $CLI_EVENTS events each" + else + echo "[ERROR] ❌ Total event counts differ: CLI=$CLI_EVENTS, Service=$SERVICE_EVENTS" + exit 1 + fi + else echo "[ERROR] One or both builds generated no events" exit 1