the battle is done... for now
Some checks are pending
/ setup (push) Waiting to run

This commit is contained in:
Stuart Axelbrooke 2025-11-25 23:31:46 +08:00
parent 14a24ef6d6
commit f531730a6b
14 changed files with 1778 additions and 47 deletions

View file

@ -109,6 +109,15 @@ crate.spec(
package = "toml",
version = "0.8",
)
crate.spec(
features = ["urlencode"],
package = "askama",
version = "0.14",
)
crate.spec(
package = "urlencoding",
version = "2.1",
)
crate.from_specs()
use_repo(crate, "crates")

File diff suppressed because one or more lines are too long

View file

@ -17,14 +17,18 @@ rust_binary(
)
# DataBuild library using generated prost code
# Note: Templates are embedded inline in web/templates.rs using Askama's in_doc feature
# because Bazel's sandbox doesn't support file-based Askama templates properly.
rust_library(
name = "lib",
srcs = glob(["**/*.rs"]) + [
":generate_databuild_rust",
],
crate_root = "lib.rs",
edition = "2021",
visibility = ["//visibility:public"],
deps = [
"@crates//:askama",
"@crates//:axum",
"@crates//:prost",
"@crates//:prost-types",
@ -33,13 +37,14 @@ rust_library(
"@crates//:schemars",
"@crates//:serde",
"@crates//:serde_json",
"@crates//:sha2",
"@crates//:tokio",
"@crates//:toml",
"@crates//:tower",
"@crates//:tower-http",
"@crates//:tracing",
"@crates//:urlencoding",
"@crates//:uuid",
"@crates//:sha2",
],
)

View file

@ -306,7 +306,10 @@ impl BuildState {
vec![]
}
pub(crate) fn handle_job_run_heartbeat(&mut self, event: &JobRunHeartbeatEventV1) -> Vec<Event> {
pub(crate) fn handle_job_run_heartbeat(
&mut self,
event: &JobRunHeartbeatEventV1,
) -> Vec<Event> {
let job_run = self.job_runs.remove(&event.job_run_id).expect(&format!(
"BUG: Job run {} must exist when heartbeat received",
event.job_run_id

View file

@ -119,7 +119,11 @@ impl BuildState {
}
/// Register a want in the wants_for_partition inverted index
pub(crate) fn register_want_for_partitions(&mut self, want_id: &str, partition_refs: &[PartitionRef]) {
pub(crate) fn register_want_for_partitions(
&mut self,
want_id: &str,
partition_refs: &[PartitionRef],
) {
for pref in partition_refs {
let want_ids = self
.wants_for_partition
@ -143,7 +147,10 @@ impl BuildState {
}
#[cfg(test)]
pub(crate) fn with_partitions(self, old_partitions: BTreeMap<String, crate::PartitionDetail>) -> Self {
pub(crate) fn with_partitions(
self,
old_partitions: BTreeMap<String, crate::PartitionDetail>,
) -> Self {
use crate::partition_state::PartitionWithState;
let mut canonical_partitions: BTreeMap<String, Uuid> = BTreeMap::new();

View file

@ -3,19 +3,23 @@
//! Methods for transitioning partitions between states (Building, Live, Failed,
//! UpstreamBuilding, UpForRetry, UpstreamFailed) and managing downstream dependencies.
use crate::PartitionRef;
use crate::partition_state::{
BuildingPartitionRef, BuildingState, FailedPartitionRef, LivePartitionRef, Partition,
PartitionWithState,
};
use crate::util::current_timestamp;
use crate::PartitionRef;
use uuid::Uuid;
use super::BuildState;
impl BuildState {
/// Create a new partition in Building state and update indexes
pub(crate) fn create_partition_building(&mut self, job_run_id: &str, partition_ref: PartitionRef) -> Uuid {
pub(crate) fn create_partition_building(
&mut self,
job_run_id: &str,
partition_ref: PartitionRef,
) -> Uuid {
let partition =
PartitionWithState::<BuildingState>::new(job_run_id.to_string(), partition_ref.clone());
let uuid = partition.uuid;
@ -209,7 +213,10 @@ impl BuildState {
/// Transition partitions from UpstreamBuilding to UpForRetry when their upstream deps become Live.
/// This should be called when partitions become Live to check if any downstream partitions can now retry.
/// Uses the `downstream_waiting` index for O(1) lookup of affected partitions.
pub(crate) fn unblock_downstream_partitions(&mut self, newly_live_partition_refs: &[LivePartitionRef]) {
pub(crate) fn unblock_downstream_partitions(
&mut self,
newly_live_partition_refs: &[LivePartitionRef],
) {
// Collect UUIDs of partitions that might be unblocked using the inverted index
let mut uuids_to_check: Vec<Uuid> = Vec::new();
for live_ref in newly_live_partition_refs {

View file

@ -9,7 +9,7 @@ use crate::{
};
use std::collections::BTreeMap;
use super::{consts, BuildState};
use super::{BuildState, consts};
impl BuildState {
pub fn get_want(&self, want_id: &str) -> Option<WantDetail> {

View file

@ -3,10 +3,10 @@
//! Methods for transitioning wants between states and managing dependencies
//! between wants (derivative wants from dep misses).
use crate::PartitionRef;
use crate::job_run_state::JobRun;
use crate::partition_state::{FailedPartitionRef, LivePartitionRef, Partition};
use crate::want_state::{FailedWantId, SuccessfulWantId, Want};
use crate::PartitionRef;
use super::BuildState;

View file

@ -13,8 +13,8 @@ use lib::orchestrator::{Orchestrator, OrchestratorConfig};
#[command(name = "databuild")]
#[command(about = "DataBuild CLI - Build system for data pipelines", long_about = None)]
struct Cli {
/// Server URL (default: http://localhost:3000)
#[arg(long, default_value = "http://localhost:3000", global = true)]
/// Server URL
#[arg(long, default_value = "http://localhost:3538", global = true)]
server: String,
#[command(subcommand)]
@ -26,7 +26,7 @@ enum Commands {
/// Start the DataBuild HTTP server
Serve {
/// Port to listen on
#[arg(long, default_value = "3000")]
#[arg(long, default_value = "3538")]
port: u16,
/// Database URL (default: :memory: for in-memory SQLite)

View file

@ -1,17 +1,23 @@
use crate::build_event_log::BELStorage;
use crate::build_state::BuildState;
use crate::commands::Command;
use crate::web::templates::{
BaseContext, HomePage, JobRunDetailPage, JobRunDetailView, JobRunsListPage,
PartitionDetailPage, PartitionDetailView, PartitionsListPage, WantDetailPage, WantDetailView,
WantsListPage,
};
use crate::{
CancelWantRequest, CreateWantRequest, CreateWantResponse, GetWantRequest, GetWantResponse,
ListJobRunsRequest, ListJobRunsResponse, ListPartitionsRequest, ListPartitionsResponse,
ListWantsRequest, ListWantsResponse,
ListWantsRequest, ListWantsResponse, PartitionStatusCode,
};
use askama::Template;
use axum::{
Json, Router,
extract::{Path, Query, Request, State},
http::{HeaderValue, Method, StatusCode},
middleware::{self, Next},
response::{IntoResponse, Response},
response::{Html, IntoResponse, Response},
routing::{delete, get, post},
};
use std::sync::{
@ -80,7 +86,7 @@ async fn update_last_request_time(
pub fn create_router(state: AppState) -> Router {
// Configure CORS for web app development
let cors = CorsLayer::new()
.allow_origin("http://localhost:3000".parse::<HeaderValue>().unwrap())
.allow_origin("http://localhost:3538".parse::<HeaderValue>().unwrap())
.allow_methods([Method::GET, Method::POST, Method::DELETE, Method::OPTIONS])
.allow_headers([
axum::http::header::CONTENT_TYPE,
@ -90,15 +96,21 @@ pub fn create_router(state: AppState) -> Router {
Router::new()
// Health check
.route("/health", get(health))
// Want endpoints
.route("/api/wants", get(list_wants))
// HTML pages
.route("/", get(home_page))
.route("/wants", get(wants_list_page))
.route("/wants/:id", get(want_detail_page))
.route("/partitions", get(partitions_list_page))
.route("/partitions/*id", get(partition_detail_page))
.route("/job_runs", get(job_runs_list_page))
.route("/job_runs/:id", get(job_run_detail_page))
// JSON API endpoints
.route("/api/wants", get(list_wants_json))
.route("/api/wants", post(create_want))
.route("/api/wants/:id", get(get_want))
.route("/api/wants/:id", get(get_want_json))
.route("/api/wants/:id", delete(cancel_want))
// Partition endpoints
.route("/api/partitions", get(list_partitions))
// Job run endpoints
.route("/api/job_runs", get(list_job_runs))
.route("/api/partitions", get(list_partitions_json))
.route("/api/job_runs", get(list_job_runs_json))
// Add CORS middleware
.layer(cors)
// Add middleware to track request time
@ -138,7 +150,251 @@ impl ErrorResponse {
}
// ============================================================================
// Handlers
// HTML Page Handlers
// ============================================================================
/// Home page
async fn home_page(State(state): State<AppState>) -> impl IntoResponse {
let build_state = match state.build_state.read() {
Ok(s) => s,
Err(_) => return Html("<h1>Error: state lock poisoned</h1>".to_string()).into_response(),
};
// Count active wants (not successful or canceled)
let active_wants_count = build_state
.list_wants(&ListWantsRequest::default())
.data
.iter()
.filter(|w| {
w.status
.as_ref()
.map(|s| s.name != "Successful" && s.name != "Canceled")
.unwrap_or(true)
})
.count() as u64;
// Count active job runs (running or queued)
let active_job_runs_count = build_state
.list_job_runs(&ListJobRunsRequest::default())
.data
.iter()
.filter(|jr| {
jr.status
.as_ref()
.map(|s| s.name == "Running" || s.name == "Queued")
.unwrap_or(false)
})
.count() as u64;
// Count live partitions
let live_partitions_count = build_state
.list_partitions(&ListPartitionsRequest::default())
.data
.iter()
.filter(|p| {
p.status
.as_ref()
.map(|s| s.code == PartitionStatusCode::PartitionLive as i32)
.unwrap_or(false)
})
.count() as u64;
let template = HomePage {
base: BaseContext::default(),
active_wants_count,
active_job_runs_count,
live_partitions_count,
};
match template.render() {
Ok(html) => Html(html).into_response(),
Err(e) => {
tracing::error!("Template render error: {}", e);
Html(format!("<h1>Template error: {}</h1>", e)).into_response()
}
}
}
/// Wants list page
async fn wants_list_page(
State(state): State<AppState>,
Query(params): Query<ListWantsRequest>,
) -> impl IntoResponse {
let build_state = match state.build_state.read() {
Ok(s) => s,
Err(_) => return Html("<h1>Error: state lock poisoned</h1>".to_string()).into_response(),
};
let response = build_state.list_wants(&params);
let template = WantsListPage {
base: BaseContext::default(),
wants: response
.data
.into_iter()
.map(WantDetailView::from)
.collect(),
page: response.page,
page_size: response.page_size,
total_count: response.match_count,
};
match template.render() {
Ok(html) => Html(html).into_response(),
Err(e) => {
tracing::error!("Template render error: {}", e);
Html(format!("<h1>Template error: {}</h1>", e)).into_response()
}
}
}
/// Want detail page
async fn want_detail_page(
State(state): State<AppState>,
Path(want_id): Path<String>,
) -> impl IntoResponse {
let build_state = match state.build_state.read() {
Ok(s) => s,
Err(_) => return Html("<h1>Error: state lock poisoned</h1>".to_string()).into_response(),
};
match build_state.get_want(&want_id) {
Some(want) => {
let template = WantDetailPage {
base: BaseContext::default(),
want: WantDetailView::from(want),
};
match template.render() {
Ok(html) => Html(html).into_response(),
Err(e) => Html(format!("<h1>Template error: {}</h1>", e)).into_response(),
}
}
None => (
StatusCode::NOT_FOUND,
Html("<h1>Want not found</h1>".to_string()),
)
.into_response(),
}
}
/// Partitions list page
async fn partitions_list_page(
State(state): State<AppState>,
Query(params): Query<ListPartitionsRequest>,
) -> impl IntoResponse {
let build_state = match state.build_state.read() {
Ok(s) => s,
Err(_) => return Html("<h1>Error: state lock poisoned</h1>".to_string()).into_response(),
};
let response = build_state.list_partitions(&params);
let template = PartitionsListPage {
base: BaseContext::default(),
partitions: response
.data
.into_iter()
.map(PartitionDetailView::from)
.collect(),
page: response.page,
page_size: response.page_size,
total_count: response.match_count,
};
match template.render() {
Ok(html) => Html(html).into_response(),
Err(e) => Html(format!("<h1>Template error: {}</h1>", e)).into_response(),
}
}
/// Partition detail page
async fn partition_detail_page(
State(state): State<AppState>,
Path(partition_ref): Path<String>,
) -> impl IntoResponse {
let build_state = match state.build_state.read() {
Ok(s) => s,
Err(_) => return Html("<h1>Error: state lock poisoned</h1>".to_string()).into_response(),
};
// Axum's Path extractor automatically percent-decodes the path parameter
match build_state.get_partition(&partition_ref) {
Some(partition) => {
let template = PartitionDetailPage {
base: BaseContext::default(),
partition: PartitionDetailView::from(partition),
};
match template.render() {
Ok(html) => Html(html).into_response(),
Err(e) => Html(format!("<h1>Template error: {}</h1>", e)).into_response(),
}
}
None => (
StatusCode::NOT_FOUND,
Html("<h1>Partition not found</h1>".to_string()),
)
.into_response(),
}
}
/// Job runs list page
async fn job_runs_list_page(
State(state): State<AppState>,
Query(params): Query<ListJobRunsRequest>,
) -> impl IntoResponse {
let build_state = match state.build_state.read() {
Ok(s) => s,
Err(_) => return Html("<h1>Error: state lock poisoned</h1>".to_string()).into_response(),
};
let response = build_state.list_job_runs(&params);
let template = JobRunsListPage {
base: BaseContext::default(),
job_runs: response
.data
.into_iter()
.map(JobRunDetailView::from)
.collect(),
page: response.page,
page_size: response.page_size,
total_count: response.match_count,
};
match template.render() {
Ok(html) => Html(html).into_response(),
Err(e) => Html(format!("<h1>Template error: {}</h1>", e)).into_response(),
}
}
/// Job run detail page
async fn job_run_detail_page(
State(state): State<AppState>,
Path(job_run_id): Path<String>,
) -> impl IntoResponse {
let build_state = match state.build_state.read() {
Ok(s) => s,
Err(_) => return Html("<h1>Error: state lock poisoned</h1>".to_string()).into_response(),
};
match build_state.get_job_run(&job_run_id) {
Some(job_run) => {
let template = JobRunDetailPage {
base: BaseContext::default(),
job_run: JobRunDetailView::from(job_run),
};
match template.render() {
Ok(html) => Html(html).into_response(),
Err(e) => Html(format!("<h1>Template error: {}</h1>", e)).into_response(),
}
}
None => (
StatusCode::NOT_FOUND,
Html("<h1>Job run not found</h1>".to_string()),
)
.into_response(),
}
}
// ============================================================================
// JSON API Handlers
// ============================================================================
/// Health check endpoint
@ -146,12 +402,11 @@ async fn health() -> impl IntoResponse {
(StatusCode::OK, "OK")
}
/// List all wants
async fn list_wants(
/// List all wants (JSON)
async fn list_wants_json(
State(state): State<AppState>,
Query(params): Query<ListWantsRequest>,
) -> impl IntoResponse {
// Read from shared build state
let build_state = match state.build_state.read() {
Ok(state) => state,
Err(e) => {
@ -170,9 +425,11 @@ async fn list_wants(
(StatusCode::OK, Json(response)).into_response()
}
/// Get a specific want by ID
async fn get_want(State(state): State<AppState>, Path(want_id): Path<String>) -> impl IntoResponse {
// Read from shared build state
/// Get a specific want by ID (JSON)
async fn get_want_json(
State(state): State<AppState>,
Path(want_id): Path<String>,
) -> impl IntoResponse {
let build_state = match state.build_state.read() {
Ok(state) => state,
Err(e) => {
@ -320,12 +577,11 @@ async fn cancel_want(
}
}
/// List all partitions
async fn list_partitions(
/// List all partitions (JSON)
async fn list_partitions_json(
State(state): State<AppState>,
Query(params): Query<ListPartitionsRequest>,
) -> impl IntoResponse {
// Read from shared build state
let build_state = match state.build_state.read() {
Ok(state) => state,
Err(e) => {
@ -344,12 +600,11 @@ async fn list_partitions(
(StatusCode::OK, Json(response)).into_response()
}
/// List all job runs
async fn list_job_runs(
/// List all job runs (JSON)
async fn list_job_runs_json(
State(state): State<AppState>,
Query(params): Query<ListJobRunsRequest>,
) -> impl IntoResponse {
// Read from shared build state
let build_state = match state.build_state.read() {
Ok(state) => state,
Err(e) => {

View file

@ -13,6 +13,7 @@ pub mod orchestrator;
mod partition_state;
mod util;
mod want_state;
pub mod web;
// Include generated protobuf code
include!("databuild.rs");

6
databuild/web/mod.rs Normal file
View file

@ -0,0 +1,6 @@
//! Web templates and handlers for the DataBuild dashboard
//!
//! Server-side rendered HTML using Askama templates with CSS View Transitions
//! for smooth navigation. No JavaScript framework required.
pub mod templates;

1316
databuild/web/templates.rs Normal file

File diff suppressed because it is too large Load diff

View file

@ -9,9 +9,9 @@ The service provides two primary capabilities:
## Correctness Strategy
- Rely on databuild.proto, call same shared code in core
- Fully asserted type safety from core to service to web app
- Core -- databuild.proto --> service -- openapi --> web app
- No magic strings (how? protobuf doesn't have consts. enums values? code gen over yaml?)
- Fully asserted type safety from core to service to web app via Askama compile-time template checking
- `databuild.proto` → prost (Rust structs) → Askama templates (compile-checked field access) → HTML
- No magic strings: enum variants and field names are checked at compile time
## Cross-Graph Coordination
Services expose the `GraphService` API for cross-graph dependency management:
@ -29,11 +29,39 @@ needed by the [web app](#web-app).
[Notes about details, context, and askama views.](https://claude.ai/share/76622c1c-7489-496e-be81-a64fef24e636)
## Web App
The web app visualizes databuild application state via features like listing past builds, job statistics,
partition liveness, build request status, etc. This section specifies the hierarchy of functions of the web app. Pages
The web app visualizes databuild application state via features like listing past builds, job statistics,
partition liveness, build request status, etc. This section specifies the hierarchy of functions of the web app. Pages
are described in visual order (generally top to bottom).
General requirements:
### Implementation Strategy: Pure MPA with View Transitions
The web app is a traditional multi-page application (MPA) using server-side rendering. No JavaScript framework.
**Stack:**
- **Askama**: Compile-time checked HTML templates in Rust
- **CSS View Transitions API**: Smooth navigation animations between pages
- **Plain HTML forms**: For mutations (create want, cancel job, etc.)
**Why this approach:**
- **Fast**: No JS bundle to download/parse; HTML streams directly from server
- **Simple**: Single Rust codebase; no frontend build pipeline
- **Type-safe**: Askama checks template field access at compile time
- **Easy to change**: Templates are just HTML; Rust compiler catches breaking changes
**View Transitions** (supported in Chrome 126+, Safari 18.2+, Firefox 144+) provide SPA-like smooth
navigation without JavaScript. The browser snapshots the old page, loads the new one, and animates
matching elements:
```css
@view-transition { navigation: auto; }
.want-card { view-transition-name: want-card; }
```
**Trade-off**: No partial page updates. Pagination and navigation reload the full page, but view
transitions make this feel instant. For live updates, users refresh manually or we add
`<meta http-equiv="refresh">` on status pages.
### General Requirements
- Nav at top of page
- DataBuild logo in top left
- Navigation links at the top allowing navigation to each list page:
@ -43,9 +71,7 @@ General requirements:
- Triggers list page
- Build event log page
- Graph label at top right
- Search box for finding builds, jobs, and partitions (needs a new service API?)
The site is implemented via Aksama templating and HTMX for dynamic updates.
- Search box for finding builds, jobs, and partitions
### Home Page
Jumping off point to navigate and build.