Update docs

README.md

@@ -5,7 +5,7 @@ DataBuild is a trivially-deployable, partition-oriented, declarative data build
 
 DataBuild is for teams at data-driven orgs who need reliable, flexible, and correct data pipelines and are tired of manually orchestrating complex dependency graphs. You define Jobs (that take input data partitions and produce output partitions), compose them into Graphs (partition dependency networks), and DataBuild handles the rest. Just ask it to build a partition, and databuild handles resolving the jobs that need to run, planning execution order, running builds concurrently, and tracking and exposing build progress. Instead of writing orchestration code that breaks when dependencies change, you focus on the data transformations while DataBuild ensures your pipelines are correct, observable, and reliable.
 
-For important context, check out [DESIGN.md](./DESIGN.md). Also, check out [`databuild.proto`](./databuild/databuild.proto) for key system interfaces. Key features:
+For important context, check out [DESIGN.md](./DESIGN.md), along with designs in [design/](./design/). Also, check out [`databuild.proto`](./databuild/databuild.proto) for key system interfaces. Key features:
 
 - **Declarative dependencies** - Ask for data, get data. Define partition dependencies and DataBuild automatically plans what jobs to run and when.
 

databuild/dashboard/README.md

@@ -0,0 +1,4 @@
+
+# Dashboard
+
+A dashboard for viewing past build status, current running builds, etc. Extremely prototyped right now.

databuild/dashboard/pages.ts

@@ -311,6 +311,9 @@ export const BuildStatus: TypedComponent<BuildStatusAttrs> = {
   error: null as string | null,
   partitionStatuses: new Map<string, DashboardPartition>(),
   buildId: '',
+  mermaidDiagram: null as string | null,
+  mermaidLoading: false,
+  mermaidError: null as string | null,
 
   oninit(vnode: m.Vnode<BuildStatusAttrs>) {
     this.buildId = vnode.attrs.id;
@@ -351,6 +354,11 @@ export const BuildStatus: TypedComponent<BuildStatusAttrs> = {
         }
       }
       
+      // Load mermaid diagram if we don't have it yet
+      if (!this.mermaidDiagram && !this.mermaidLoading) {
+        this.loadMermaidDiagram();
+      }
+      
       this.loading = false;
       m.redraw();
     } catch (error) {
@@ -361,6 +369,37 @@ export const BuildStatus: TypedComponent<BuildStatusAttrs> = {
     }
   },
 
+  async loadMermaidDiagram() {
+    try {
+      this.mermaidLoading = true;
+      this.mermaidError = null;
+      m.redraw();
+
+      const service = DashboardService.getInstance();
+      const diagram = await service.getMermaidDiagram(this.buildId);
+      
+      if (diagram) {
+        this.mermaidDiagram = diagram;
+        // Trigger mermaid to render the diagram after the DOM updates
+        setTimeout(() => {
+          if (typeof window !== 'undefined' && (window as any).mermaid) {
+            (window as any).mermaid.init();
+          }
+        }, 100);
+      } else {
+        this.mermaidError = 'No job graph available for this build';
+      }
+      
+      this.mermaidLoading = false;
+      m.redraw();
+    } catch (error) {
+      console.error('Failed to load mermaid diagram:', error);
+      this.mermaidError = error instanceof Error ? error.message : 'Failed to load diagram';
+      this.mermaidLoading = false;
+      m.redraw();
+    }
+  },
+
   startPolling() {
     // Use different poll intervals based on build status
     const isActive = this.data?.status_name === 'EXECUTING' ||
@@ -414,6 +453,15 @@ export const BuildStatus: TypedComponent<BuildStatusAttrs> = {
 
     const build = this.data;
 
+    const timelineData = [
+      {stage: 'Build Requested', time: build.requested_at, icon: '🕚'},
+      ...(build.started_at ? [{stage: 'Build Started', time: build.started_at, icon: '▶️'}] : []),
+      // ...(this.data.events as BuildEvent[]).filter(ev => ev.job_event !== null).map((ev) => ({
+      //   stage: ev.job_event.status_name, time: ev.timestamp, icon: '🙃'
+      // })),
+      ...(build.completed_at ? [{stage: 'Build Completed', time: build.completed_at, icon: '✅'}] : []),
+    ];
+
     return m('div.container.mx-auto.p-4', [
       // Build Header
       m('.build-header.mb-6', [
@@ -432,12 +480,12 @@ export const BuildStatus: TypedComponent<BuildStatusAttrs> = {
           ]),
           m('.stat.bg-base-100.shadow.rounded-lg.p-4', [
             m('.stat-title', 'Jobs'),
-            m('.stat-value.text-2xl', `${build.completed_jobs}/${build.total_jobs}`),
+            m('.stat-value.text-2xl', `${build.completed_jobs}`),
             m('.stat-desc', 'completed')
           ]),
           m('.stat.bg-base-100.shadow.rounded-lg.p-4', [
             m('.stat-title', 'Duration'),
-            m('.stat-value.text-2xl', build.duration_ms ? formatDuration(build.duration_ms) : '—'),
+            m('.stat-value.text-2xl', (build.completed_at - build.started_at) ? formatDuration((build.completed_at - build.started_at)) : '—'),
             m('.stat-desc', build.started_at ? formatDateTime(build.started_at) : 'Not started')
           ])
         ])
@@ -504,31 +552,74 @@ export const BuildStatus: TypedComponent<BuildStatusAttrs> = {
                 m('.metric-label.text-sm.opacity-60', 'Total Jobs')
               ])
             ]),
-            m('.build-timeline.mt-6', [
-              m('.timeline.text-sm', [
-                m('.timeline-item', [
-                  m('.timeline-marker.text-primary', '●'),
-                  m('.timeline-content', [
-                    m('.font-medium', 'Requested'),
-                    m('.opacity-60', formatDateTime(build.requested_at))
-                  ])
+
+            m('ul.timeline.mx-auto',timelineData.map((item) => {
+              return m('li', [
+                ...(item.stage === 'Build Requested' ? [] : [m("hr")]),
+                m('div.font-medium.timeline-middle', item.icon),
+                m("div.timeline-box.timeline-end", {
+                    style: {
+                      '--timeline-color': item.stage === 'Requested' ? '#1976d2' : '#6b7280'
+                    }
+                  }, [
+                    m('span.flex.flex-col.gap-2', {
+                      class: 'timeline-point',
+                    }, [
+                      m('.font-medium', item.stage),
+                      m('.text-xs.opacity-60', formatDateTime(item.time)),
+                    ])
+                  ]
+                ),
+                ...(item.stage === 'Build Completed' ? [] : [m("hr")]),
+              ]
+            )
+
+            })),
+
+          ])
+        ]),
+
+        // Build Graph Visualization
+        m('.build-graph.card.bg-base-100.shadow-xl', [
+          m('.card-body', [
+            m('.flex.justify-between.items-center.mb-4', [
+              m('h2.card-title.text-xl', 'Build Graph'),
+              this.mermaidDiagram && m('button.btn.btn-sm.btn-outline', {
+                onclick: () => this.loadMermaidDiagram()
+              }, 'Refresh Diagram')
+            ]),
+            this.mermaidLoading ? 
+              m('.flex.justify-center.items-center.h-32', [
+                m('span.loading.loading-spinner.loading-lg'),
+                m('span.ml-4', 'Loading diagram...')
+              ]) :
+            this.mermaidError ?
+              m('.alert.alert-warning', [
+                m('svg.stroke-current.shrink-0.h-6.w-6', {
+                  fill: 'none',
+                  viewBox: '0 0 24 24'
+                }, [
+                  m('path', {
+                    'stroke-linecap': 'round',
+                    'stroke-linejoin': 'round',
+                    'stroke-width': '2',
+                    'd': 'M12 9v2m0 4h.01m-6.938 4h13.856c1.54 0 2.502-1.667 1.732-2.5L13.732 4c-.77-.833-1.964-.833-2.732 0L3.732 16.5c-.77.833.192 2.5 1.732 2.5z'
+                  })
                 ]),
-                build.started_at && m('.timeline-item', [
-                  m('.timeline-marker.text-info', '●'),
-                  m('.timeline-content', [
-                    m('.font-medium', 'Started'),
-                    m('.opacity-60', formatDateTime(build.started_at))
-                  ])
-                ]),
-                build.completed_at && m('.timeline-item', [
-                  m('.timeline-marker.text-success', '●'),
-                  m('.timeline-content', [
-                    m('.font-medium', 'Completed'),
-                    m('.opacity-60', formatDateTime(build.completed_at))
-                  ])
+                m('span', this.mermaidError),
+                m('div', [
+                  m('button.btn.btn-sm.btn-outline', {
+                    onclick: () => this.loadMermaidDiagram()
+                  }, 'Retry')
                 ])
-              ].filter(Boolean))
-            ])
+              ]) :
+            this.mermaidDiagram ?
+              m('.mermaid-container.w-full.overflow-x-auto', [
+                m('pre.mermaid.text-center', {
+                  key: `mermaid-${this.buildId}` // Force re-render when buildId changes
+                }, this.mermaidDiagram)
+              ]) :
+              m('.text-center.py-8.text-base-content.opacity-60', 'No graph visualization available')
           ])
         ])
       ])

databuild/dashboard/services.ts

@@ -320,6 +320,32 @@ export class DashboardService {
       return null;
     }
   }
+
+  async getMermaidDiagram(buildId: string): Promise<string | null> {
+    try {
+      const url = `/api/v1/builds/${buildId}/mermaid`;
+      
+      const response = await fetch(url);
+      if (!response.ok) {
+        if (response.status === 404) {
+          return null; // Build not found or no job graph
+        }
+        throw new Error(`HTTP ${response.status}: ${response.statusText}`);
+      }
+      
+      const data = await response.json();
+      
+      // Validate response structure
+      if (typeof data === 'object' && data !== null && 'mermaid' in data && typeof data.mermaid === 'string') {
+        return data.mermaid;
+      }
+      
+      throw new Error('Invalid mermaid response structure');
+    } catch (error) {
+      console.error('Failed to fetch mermaid diagram:', error);
+      return null;
+    }
+  }
 }
 
 // Polling manager with Page Visibility API integration

databuild/service/handlers.rs

@@ -1679,4 +1679,77 @@ pub async fn get_job_run_metrics(
             ))
         }
     }
+}
+
+/// Request for build mermaid diagram endpoint
+#[derive(Deserialize, JsonSchema)]
+pub struct BuildMermaidRequest {
+    pub build_request_id: String,
+}
+
+/// Response for build mermaid diagram endpoint
+#[derive(serde::Serialize, JsonSchema)]
+pub struct BuildMermaidResponse {
+    pub mermaid: String,
+}
+
+/// Get Mermaid diagram for a specific build request ID
+pub async fn get_build_mermaid_diagram(
+    State(service): State<ServiceState>,
+    Path(BuildMermaidRequest { build_request_id }): Path<BuildMermaidRequest>,
+) -> Result<Json<BuildMermaidResponse>, (StatusCode, Json<ErrorResponse>)> {
+    info!("Generating mermaid diagram for build request {}", build_request_id);
+    
+    // Get build events for this build request
+    let events = match service.event_log.get_build_request_events(&build_request_id, None).await {
+        Ok(events) => events,
+        Err(e) => {
+            error!("Failed to get build events for {}: {}", build_request_id, e);
+            return Err((
+                StatusCode::INTERNAL_SERVER_ERROR,
+                Json(ErrorResponse {
+                    error: format!("Failed to get build events: {}", e),
+                }),
+            ));
+        }
+    };
+    
+    if events.is_empty() {
+        return Err((
+            StatusCode::NOT_FOUND,
+            Json(ErrorResponse {
+                error: "Build request not found".to_string(),
+            }),
+        ));
+    }
+    
+    // Find job graph event to get the graph structure
+    let job_graph = events.iter()
+        .find_map(|event| {
+            match &event.event_type {
+                Some(crate::build_event::EventType::JobGraphEvent(graph_event)) => {
+                    graph_event.job_graph.as_ref()
+                }
+                _ => None,
+            }
+        });
+    
+    match job_graph {
+        Some(graph) => {
+            // Generate mermaid diagram with current status
+            let mermaid_diagram = mermaid_utils::generate_mermaid_with_status(graph, &events);
+            
+            Ok(Json(BuildMermaidResponse {
+                mermaid: mermaid_diagram,
+            }))
+        }
+        None => {
+            Err((
+                StatusCode::NOT_FOUND,
+                Json(ErrorResponse {
+                    error: "No job graph found for this build request".to_string(),
+                }),
+            ))
+        }
+    }
 }

databuild/service/mod.rs

@@ -47,18 +47,6 @@ pub struct BuildRequestResponse {
     pub build_request_id: String,
 }
 
-#[derive(Debug, Serialize, Deserialize, JsonSchema)]
-pub struct BuildStatusResponse {
-    pub build_request_id: String,
-    pub status: String,
-    pub requested_partitions: Vec<String>,
-    pub created_at: i64,
-    pub updated_at: i64,
-    pub events: Vec<BuildEventSummary>,
-    pub job_graph: Option<serde_json::Value>,
-    pub mermaid_diagram: Option<String>,
-}
-
 #[derive(Debug, Serialize, Deserialize, JsonSchema)]
 pub struct BuildEventSummary {
     pub event_id: String,
@@ -272,6 +260,7 @@ impl BuildGraphService {
             .api_route("/api/v1/builds", post(handlers::submit_build_request))
             .api_route("/api/v1/builds", get(handlers::list_builds_repository))
             .api_route("/api/v1/builds/:build_request_id", get(handlers::get_build_detail))
+            .api_route("/api/v1/builds/:build_request_id/mermaid", get(handlers::get_build_mermaid_diagram))
             .api_route("/api/v1/builds/:build_request_id", delete(handlers::cancel_build_repository))
             .api_route("/api/v1/partitions", get(handlers::list_partitions_repository))
             .api_route("/api/v1/partitions/:partition_ref", get(handlers::get_partition_detail))

design/why-databuild.md

@@ -1,11 +1,84 @@
 
-# Why DataBuild?
+# Why DataBuild? A Case for Declarative Data Orchestration
 
-(work in progress)
+Bullet points that should eventually become a blog post.
 
-Why?
-- Orchestration logic changes all the time, better to not write it directly
-- Declarative -> Compile time correctness (e.g. can detect when no job produces a partition pattern)
-- Compartmentalized jobs + data deps -> Simplicity and compartmentalization of complexity
-- Bazel based -> Easy to deploy, maintain, and update
+## Introduction
+- **The Vision**: What if data engineers could iterate fearlessly on massive pipelines? If individual engineers could onboard and support 10x as many datasets?
+- **The Problem**: Most orchestration abstractions are brittle - moving quickly and correctly without complete product certainty is difficult
+- **The Reality**: Teams either move fast and break things, or move slowly to avoid breaking things
+- **The Promise**: Declarative data orchestration enables both speed and correctness
+- **The Inspiration**: Learning from the declarative evolutions in Bazel, SQL, and Kubernetes
 
+## The Hidden Costs of Data Interfaces
+
+### Coupling by Obscurity
+- Data interfaces hide critical dependencies between jobs
+- Violates fundamental software design principles:
+    - **Dependencies**: Can't understand impact in isolation
+    - **Obscurity**: Critical coupling information isn't visible in code
+    - **Change amplification**: Simple changes require modifications everywhere
+- Example: Bug fix in dataset A breaks jobs B, C, D... but you don't know until runtime
+
+### The Orchestration Trap
+- Engineers spend too much time writing, updating, and manually testing orchestration
+- Orchestration code is:
+    - Constantly changing as requirements evolve
+    - Nearly impossible to test meaningfully
+    - Brittle and breaks when the dependency graph changes
+
+## The Declarative Alternative
+
+### Learning from Proven Systems
+**Bazel**: Declare targets and dependencies → system handles build orchestration
+**SQL**: Declare what data you want → query planner handles execution strategy  
+**Kubernetes**: Declare desired state → controllers handle deployment orchestration
+
+### Inversion of Control for Data
+- Engineers declare **what**: jobs and data dependencies
+- System handles **how**: execution order, parallelization, failure recovery
+- Enables **local reasoning**: understand jobs in isolation
+- Supports **fearless iteration**: changes automatically propagate correctly
+
+### Continuous Reconciliation
+- Triggers periodically ask: "ensure all expected partitions exist"
+- DataBuild determines what's missing, stale, or needs rebuilding
+- System maintains desired state without manual intervention
+- Self-healing pipelines that adapt to changing upstream data
+
+## Operational Simplicity
+
+### Stateless by Design
+- Each build request is independent and ephemeral
+- Append-only event log vs. complex mutable orchestrator state
+- No database migrations or careful state preservation across versions
+
+### Deployment as Code Compilation
+- Following Bazel's model: build binary, ship it
+- Auto-generated deployment configs (Helm charts, etc.)
+- Version updates are the norm, not the exception
+
+### Separation of Concerns
+- **DataBuild**: Dependency resolution + execution planning
+- **External systems**: Scheduling (cron/triggers), infrastructure (Kubernetes)
+- **Result**: Operational complexity focused where it belongs
+
+## The DataBuild Vision
+
+### Core Tenets
+- No dependency knowledge necessary to materialize data
+- Only local dependency knowledge needed to develop
+- Explicit coupling via declared data dependencies
+- Automatic orchestration delegation
+
+### What This Enables
+- **Fearless iteration**: Change any part of a large graph confidently
+- **Trivial deployment**: Single binary updates, no complex state management
+- **Automatic correctness**: System prevents composition bugs at "compile time"
+- **Scalable development**: Near-zero marginal effort for new datasets
+
+## Conclusion
+- Data engineering doesn't have to be this hard
+- Declarative approaches have transformed other domains
+- DataBuild brings these proven patterns to data orchestration
+- The future: engineers focus on business logic, systems handle the complexity