Update databuild.proto for v2

databuild/databuild.proto

@@ -50,7 +50,7 @@ message JobConfigureResponse { repeated JobConfig configs = 1; }
 
 // Implemented by the job.cfg bazel rule
 service JobConfigure {
-  rpc Configure(JobConfigureRequest) returns (JobConfigureResponse);
+ rpc Configure(JobConfigureRequest) returns (JobConfigureResponse);
 }
 
 //
@@ -86,7 +86,7 @@ message JobExecuteResponse { repeated PartitionManifest manifests = 1; }
 
 // Implemented by the job.exec bazel rule
 service JobExecute {
-  rpc Execute(JobExecuteRequest) returns (JobExecuteResponse);
+ rpc Execute(JobExecuteRequest) returns (JobExecuteResponse);
 }
 
 ///////////////////////////////////////////////////////////////////////////////////////////////
@@ -119,7 +119,7 @@ message GraphLookupResponse { repeated TaskRef task_refs = 1; }
 
 // Implemented per graph
 service GraphLookup {
-  rpc Lookup(GraphLookupRequest) returns (GraphLookupResponse);
+ rpc Lookup(GraphLookupRequest) returns (GraphLookupResponse);
 }
 
 // Request message for graph analyze service
@@ -202,23 +202,46 @@ enum JobStatus {
 }
 
 // Build request lifecycle
-enum BuildRequestStatus {
+enum BuildRequestStatusCode {
+  // Not good
   BUILD_REQUEST_UNKNOWN = 0;
-  BUILD_REQUEST_RECEIVED = 1;         // Build request received and queued
+  // Build request received
-  BUILD_REQUEST_PLANNING = 2;         // Graph analysis in progress
+  BUILD_REQUEST_RECEIVED = 1;
-  BUILD_REQUEST_ANALYSIS_COMPLETED = 7; // Graph analysis completed successfully
+  // Graph analysis in progress
-  BUILD_REQUEST_EXECUTING = 3;        // Jobs are being executed
+  BUILD_REQUEST_PLANNING = 2;
-  BUILD_REQUEST_COMPLETED = 4;        // All requested partitions built
+  // Graph analysis completed successfully
-  BUILD_REQUEST_FAILED = 5;           // Build request failed
+  BUILD_REQUEST_ANALYSIS_COMPLETED = 7;
-  BUILD_REQUEST_CANCELLED = 6;        // Build request cancelled
+  // Jobs are being executed
+  BUILD_REQUEST_EXECUTING = 3;
+  // All requested partitions built
+  BUILD_REQUEST_COMPLETED = 4;
+  // Build precondition failed (e.g. required external data was not available)
+  BUILD_REQUEST_PRECONDITION_FAILED = 8;
+  // Build request failed
+  BUILD_REQUEST_FAILED = 5;
+  // Build request cancelled
+  BUILD_REQUEST_CANCELLED = 6;
+}
+
+message BuildRequestStatus {
+  // Enum for programmatic use
+  BuildRequestStatusCode code = 1;
+  // Human readable string
+  string name = 2;
 }
 
 // Build request lifecycle event
 message BuildRequestEvent {
-  BuildRequestStatus status_code = 1;     // Enum for programmatic use
+  // The status that this event indicates
-  string status_name = 2;                 // Human-readable string
+  BuildRequestStatus status = 1;
+  // Output partitions requested to be built as part of this build
   repeated PartitionRef requested_partitions = 3;
-  string message = 4;                     // Optional status message
+  // Optional status message
+  string message = 4;
+  // The comment attached to the request - contains arbitrary text
+  string comment = 5;
+  // The id of the want that triggered this build
+  string want_id = 6;
 }
 
 // Partition state change event
@@ -272,20 +295,66 @@ message BuildCancelEvent {
   string reason = 1;                      // Reason for cancellation
 }
 
-// Partition Want
+message WantEvent {
-message WantSource {
+  repeated PartitionRef requested_partitions = 1;
-  // TODO
+  // Unique identifier
+  string want_id = 2;
+  // How this want was created
+  WantSource source = 3;
 }
 
 message PartitionWant {
-  PartitionRef partition_ref = 1;         // Partition being requested
+  string want_id = 1;
-  uint64 created_at = 2;                  // Server time when want registered
+  // The ref we want to materialize
-  optional uint64 data_timestamp = 3;     // Business time this partition represents
+  PartitionRef ref = 2;
-  optional uint64 ttl_seconds = 4;        // Give up after this long (from created_at)
+  // Server time when want registered
-  optional uint64 sla_seconds = 5;        // SLA violation after this long (from data_timestamp)
+  uint64 created_at = 3;
-  repeated string external_dependencies = 6; // Cross-graph dependencies
+  // Business time this partition represents
-  string want_id = 7;                     // Unique identifier
+  uint64 data_timestamp = 4;
-  WantSource source = 8;                  // How this want was created
+  // Give up after this long (from created_at)
+  optional uint64 ttl_seconds = 5;
+  // SLA violation after this long (from data_timestamp)
+  optional uint64 sla_seconds = 6;
+  // Cross-graph dependencies determined in the analysis phase triggered upon want submission
+  // These are per-partition, since wants can be partially, marginally materialized
+  repeated string external_dependencies = 7;
+}
+
+message WantSource {
+  // The source of the want
+  SourceType source_type = 1;
+
+  // TODO implement something to record want actual want source for external requests when we have real use case
+}
+
+message SourceType {
+  SourceTypeCode code = 1;
+  string name = 2;
+}
+
+enum SourceTypeCode {
+  // Manual CLI request
+  CLI_MANUAL = 0;
+  // Manual dashboard request
+  DASHBOARD_MANUAL = 1;
+  // Scheduled/triggered job
+  SCHEDULED = 2;
+  // External API call
+  API_REQUEST = 3;
+}
+
+
+// Marks a partition as tainted, so that it will be rebuilt if a data dep points to it, and will be rebuilt if a live
+// want points to it.
+message TaintEvent {
+  // The list of partitions to be tainted
+  repeated PartitionRef refs = 1;
+  // When the taint was created
+  uint64 created_at = 2;
+  // The source of the taint event
+  SourceType source_type = 3;
+  // Free text comment attached to the taint
+  string comment = 4;
 }
 
 // Individual build event
@@ -293,7 +362,6 @@ message BuildEvent {
   // Event metadata
   string event_id = 1;                    // UUID for this event
   int64 timestamp = 2;                    // Unix timestamp (nanoseconds)
-  string build_request_id = 3;            // UUID of the build request
   
   // Event type and payload (one of)
   oneof event_type {
@@ -305,6 +373,8 @@ message BuildEvent {
     PartitionInvalidationEvent partition_invalidation_event = 15;
     JobRunCancelEvent job_run_cancel_event = 16;
     BuildCancelEvent build_cancel_event = 17;
+    WantEvent want_event = 18;
+    TaintEvent taint_event = 19;
   }
 }
 
@@ -460,7 +530,7 @@ message BuildsListResponse {
 
 message BuildSummary {
   string build_request_id = 1;
-  BuildRequestStatus status_code = 2;     // Enum for programmatic use
+  BuildRequestStatusCode status_code = 2;     // Enum for programmatic use
   string status_name = 3;                 // Human-readable string
   repeated PartitionRef requested_partitions = 4;
   uint32 total_jobs = 5;
@@ -472,6 +542,7 @@ message BuildSummary {
   optional int64 completed_at = 11;
   optional int64 duration_ms = 12;
   bool cancelled = 13;
+  string comment = 14;
 }
 
 //
@@ -501,7 +572,7 @@ message BuildDetailRequest {
 
 message BuildDetailResponse {
   string build_request_id = 1;
-  BuildRequestStatus status_code = 2;     // Enum for programmatic use
+  BuildRequestStatusCode status_code = 2;     // Enum for programmatic use
   string status_name = 3;                 // Human-readable string
   repeated PartitionRef requested_partitions = 4;
   uint32 total_jobs = 5;
@@ -519,7 +590,7 @@ message BuildDetailResponse {
 
 message BuildTimelineEvent {
   int64 timestamp = 1;
-  optional BuildRequestStatus status_code = 2;  // Enum for programmatic use
+  optional BuildRequestStatusCode status_code = 2;  // Enum for programmatic use
   optional string status_name = 3;              // Human-readable string
   string message = 4;
   string event_type = 5;
@@ -641,20 +712,377 @@ message JobLogsResponse {
 }
 
 ///////////////////////////////////////////////////////////////////////////////////////////////
-// Services
+// Currently unused - implemented via HTTP REST API instead
 ///////////////////////////////////////////////////////////////////////////////////////////////
 
+// Partition Want (Future feature - currently unused)
+// message WantSource {
+//   // TODO
+// }
+
+// message PartitionWant {
+//   PartitionRef partition_ref = 1;         // Partition being requested
+//   uint64 created_at = 2;                  // Server time when want registered
+//   optional uint64 data_timestamp = 3;     // Business time this partition represents
+//   optional uint64 ttl_seconds = 4;        // Give up after this long (from created_at)
+//   optional uint64 sla_seconds = 5;        // SLA violation after this long (from data_timestamp)
+//   repeated string external_dependencies = 6; // Cross-graph dependencies
+//   string want_id = 7;                     // Unique identifier
+//   WantSource source = 8;                  // How this want was created
+// }
+
 // Service for job configuration and graph analysis
+// service DataBuildService {
+//   // Get job configurations for the specified output references
+//   // rpc GetJobConfigs(JobConfigureRequest) returns (JobConfigureResponse) {}
+
+//   // Analyze and get the job graph for the specified output references
+//   rpc AnalyzeGraph(GraphAnalyzeRequest) returns (GraphAnalyzeResponse);
+
+//   // Execute the specified job graph (implemented by databuild)
+//   rpc Execute(JobGraph) returns (GraphExecuteResponse);
+
+//   // User-facing: build the desired partitions
+//   rpc Build(GraphBuildRequest) returns (GraphBuildResponse);
+// }
+
+
+///////////////////////////////////////////////////////////////////////////////////////////////
+// DataBuildService - v2 of service and CLI interface below
+///////////////////////////////////////////////////////////////////////////////////////////////
+
+// The service that vends all build status information
+// Core objects are:
+// - Build events - events emitted as part of the build process that indicate status/state
+// - BuildRequests - the literal request to build 1+ partitions
+// - Partitions - Atomic units of data that represent results of jobs, and act as sufficiency signals for other jobs
+// - Jobs - the units of work that build partitions (a single run of one is a JobRun)
+// - JobRuns - the specific runs of Jobs
+// - Wants - the recorded "want" to build a partition, which will be acted on ASAP
+// - Taints - invalidate built partitions, in cases where the result should not be used or should be rebuilt
+// Each of these will have a list page, and all but build events will have a summary page.
 service DataBuildService {
-//  // Get job configurations for the specified output references
+  // Build events - exposes literal events from build event log with filters
-//  rpc GetJobConfigs(JobConfigureRequest) returns (JobConfigureResponse) {}
+  rpc GetBuildEvents(ListBuildEventsRequest) returns (ListBuildEventsResponse);
 
-  // Analyze and get the job graph for the specified output references
+  // For batched requests
-  rpc AnalyzeGraph(GraphAnalyzeRequest) returns (GraphAnalyzeResponse);
+  rpc Batched(BatchedRequest) returns (BatchedResponse);
 
-  // Execute the specified job graph (implemented by databuild)
+  // BUILDS
-  rpc Execute(JobGraph) returns (GraphExecuteResponse);
+  // List the available build requests with limited metadata about them (requested partitions, status, requested time, etc)
+  rpc ListBuildRequests(ListBuildsRequest) returns (ListBuildsResponse);
+  // Get build status, summary, and paginated lists of produced partitions, and other related metadata
+  rpc GetBuildSummary(BuildSummaryRequest) returns (BuildSummaryResponse);
+  // Get a mermaid description of the build request graph with its current status rendered
+  rpc GetBuildMermaid(BuildSummaryRequest) returns (BuildMermaidResponse);
 
-  // User-facing: build the desired partitions
+  // PARTITIONS
-  rpc Build(GraphBuildRequest) returns (GraphBuildResponse);
+  // List partitions (built, building, wanted)
+  rpc ListPartitions(ListPartitionsRequest) returns (ListPartitionsResponse);
+  // Get details about a specific partition (status, created at, past builds, job runs that built or are building it, etc)
+  rpc GetPartitionsSummary(PartitionSummaryRequest) returns (PartitionSummaryResponse);
+
+  // JOBS
+  // List jobs described in the graph plus metadata (success rate, last result, last run at, etc)
+  rpc ListJobs(ListJobsRequest) returns (ListJobsResponse);
+  // Get details for a specific job
+  rpc GetJobSummary(JobSummaryRequest) returns (JobSummaryResponse);
+
+  // JOB RUNS
+  // List job runs plus basic metadata (job they ran, result, runtime, etc)
+  rpc ListJobRuns(ListJobRunsRequest) returns (ListJobRunsResponse);
+  // Get details of a specific job run (above details plus produced partitions, paginated logs, etc)
+  rpc GetJobRunSummary(JobRunSummaryRequest) returns (JobRunSummaryResponse);
+
+  // Wants
+  // List wants plus metadata (wanted partitions, created at, status)
+  rpc ListWants(ListWantsRequest) returns (ListWantsResponse);
+  // Get details for a want (above plus reasons for want being in current state, etc)
+  rpc GetWantSummary(WantSummaryRequest) returns (WantSummaryResponse);
+  // Register a want (list of partition refs, with user, reason, etc)
+  rpc PutWants(PutWantsRequest) returns (PutWantsResponse);
+
+  // Taints
+  // List taints plus metadata (tainted partitions, created at, status)
+  rpc ListTaints(ListTaintsRequest) returns (ListTaintsResponse);
+  // Summarize the requested taint
+  rpc GetTaintSummary(TaintSummaryRequest) returns (TaintSummaryResponse);
+  // Register a taint (list of partition refs, with user, reason, etc)
+  rpc PutTaints(PutTaintsRequest) returns (PutTaintsResponse);
+}
+
+message RequestContainer {
+  ListBuildEventsResponse list_build_events = 1;
+  BuildSummaryRequest build_request_status = 2;
+
+  // TODO
+}
+
+message ResponseContainer {
+  ListBuildEventsResponse list_build_events = 1;
+  BuildSummaryResponse build_request_status = 2;
+  // TODO
+}
+
+message ErrorContainer {
+  string error_message = 1;
+}
+
+message BatchedRequest {
+  map<string, RequestContainer> requests = 1;
+}
+
+message BatchedResponse {
+  map<string, ResponseContainer> responses = 1;
+  map<string, ErrorContainer> errors = 2;
+}
+
+// BEL events
+
+message ListBuildEventsRequest {
+    EventFilter filters = 1;
+
+    // Either one of the following must be provided
+    // Scrolls backwards from the specified timestamp
+    uint64 max_timestamp_ns = 2;
+    // Scrolls forward from the specified timestamp
+    uint64 min_timestamp_ns = 3;
+}
+
+message ListBuildEventsResponse {
+  // Resulting events are ordered
+  repeated BuildEvent events = 1;
+  bool has_more = 2;
+}
+
+// BUILD REQUESTS
+
+// ANDed filters
+message ListBuildsRequest {
+  // The max time the service will search until to find build requests
+  uint64 started_until = 1;
+  // Filters returned build requests those that currently have this status
+  repeated string build_status = 2;
+  // Filters build requests to those that built one of these partitions
+  repeated string built_partition = 3;
+  // Filters build requests to those that output one of these partitions (excluding those that were not explicitly
+  // requested in the build request)
+  repeated string output_partition = 4;
+  // Filters by jobs that were run as part of the build
+  repeated string run_jobs = 5;
+  // Filters by the ID of the want that triggered the build
+  repeated string triggering_want_ids = 6;
+  // Filters by contains match against build request comment
+  string comment_contains = 7;
+}
+
+// Ordered and paginated by build start time
+message ListBuildsResponse {
+  // Resulting builds
+  repeated BuildSummary builds = 1;
+  // Paging bounds for requesting next page
+  uint64 min_started = 2;
+  // Indicates if there are more to request
+  bool has_more = 3;
+}
+
+message BuildSummaryRequest {
+  string build_id = 1;
+}
+
+message BuildSummaryResponse {
+  string build_id = 1;
+
+  // Overall status of the build
+  BuildRequestStatusCode status = 2;
+  // Summary of the build
+  BuildSummary summary = 3;
+  // Partitions produced by the build
+  repeated PartitionBuildStatus partitions = 4;
+
+}
+
+message PartitionBuildStatus {
+  PartitionRef ref = 1;
+  PartitionStatus status = 2;
+}
+
+message BuildMermaidResponse {
+  string build_id = 1;
+  string mermaid = 2;
+}
+
+// PARTITIONS
+
+message ListPartitionsRequest {
+  // Optional regex filter
+  string ref_pattern = 1;
+  // Optional ORing partition status filter
+  repeated PartitionStatus partition_status = 2;
+  // Basic pagination mechanism - returns partitions sorted after the provided ref
+  string last_partition = 3;
+}
+
+message ListPartitionsResponse {
+  repeated PartitionSummaryV2 refs = 1;
+}
+
+message PartitionStatusV2 {
+  PartitionStatus code = 1;
+  string name = 2;
+}
+
+message PartitionSummaryV2 {
+  PartitionRef partition_ref = 1;
+  PartitionStatusV2 status = 2;
+  uint64 last_updated = 4;
+  uint64 last_invalidated_at = 6;
+  repeated string past_build_request = 7;
+}
+
+message PartitionSummaryRequest {
+  PartitionRef ref = 1;
+}
+
+message PartitionSummaryResponse {
+  PartitionSummaryV2 partition = 1;
+}
+
+// JOBS
+
+// No query params - if you need to paginate here something is insane or you're google
+message ListJobsRequest {}
+
+message ListJobsResponse {
+  repeated JobSummary jobs = 1;
+}
+
+message JobSummaryRequest {
+  string job_label = 1;
+}
+
+message JobSummaryResponse {
+  JobSummary job = 1;
+}
+
+// JOB RUNS
+
+// Paginates backwards
+message ListJobRunsRequest {
+  // Filters to job runs started until this point
+  uint64 started_until = 1;
+  // ORing filter matching job run IDs
+  repeated string job_run_ids = 2;
+  // ORing filters to job runs that are defined by one of these job labels
+  repeated string job_labels = 3;
+  // ORing filters to job runs that were involved in one of these build requests
+  repeated string build_reqeust_ids = 4;
+  // ORing filters to partitions produced by these job runs
+  repeated string built_partition_refs = 5;
+}
+
+message ListJobRunsResponse {
+  repeated JobRunSummary job_runs = 1;
+  uint64 min_start_at = 2;
+}
+
+message JobRunSummaryRequest {
+  string job_run_id = 1;
+}
+
+message JobRunSummaryResponse {
+  JobRunSummary job_run = 1;
+}
+
+// WANTS
+
+message ListWantsRequest {
+  // Filters the latest time the want could been requested until
+  uint64 requested_until = 1;
+  // Filters to wants whose ttl expires after ttl_until (allows querying "currently wanted"
+  uint64 ttl_until = 2;
+}
+
+message ListWantsResponse {
+  repeated PartitionWantSummary wants = 1;
+  uint64 min_requested_at = 2;
+}
+
+message LabeledPartitionBuildStatus {
+  PartitionRef ref = 1;
+  PartitionBuildStatus status = 2;
+}
+
+message PartitionWantSummary {
+  PartitionWant want = 1;
+  repeated PartitionSummary partitions = 2;
+  repeated LabeledPartitionBuildStatus external_partitions = 3;
+  string comment = 4;
+}
+
+message WantSummaryRequest {
+  string want_id = 1;
+}
+
+message WantSummaryResponse {
+  PartitionWantSummary want = 1;
+}
+
+message IndividualWantRequest {
+  PartitionRef ref = 1;
+  uint64 date_timestamp = 2;
+  uint64 ttl_seconds = 3;
+  uint64 sla_seconds = 4;
+}
+
+message PutWantsRequest {
+  repeated IndividualWantRequest wants = 1;
+  WantSource source = 2;
+  string comment = 3;
+}
+
+message CreatedWant {
+  PartitionRef ref = 1;
+  string want_id = 2;
+}
+
+message PutWantsResponse {
+  repeated CreatedWant wants = 1;
+}
+
+// TAINTS
+
+message ListTaintsRequest {
+  uint64 tainted_at_until = 1;
+}
+
+message ListTaintsResponse {
+  repeated PartitionTaintSummary taints = 1;
+  uint64 min_tainted_at = 2;
+}
+
+message PartitionTaintSummary {
+  string taint_id = 1;
+  repeated PartitionRef refs = 2;
+  uint64 tainted_at = 3;
+  SourceType source = 4;
+  string comment = 5;
+}
+
+message TaintSummaryRequest {
+  string taint_id = 1;
+}
+
+message TaintSummaryResponse {
+  PartitionTaintSummary taint = 1;
+}
+
+message PutTaintsRequest {
+  repeated PartitionRef refs = 1;
+  SourceType source = 2;
+  string comment = 3;
+}
+
+message PutTaintsResponse {
+  string taint_id = 1;
 }

design/questions.md

@@ -0,0 +1,5 @@
+
+# Questions
+
+- What happens when we deploy a new graph, and nothing builds a wanted partition?
+  - Is the interaction model between graph_a -> graph_b actually graph_a registering a want in graph_b?

plans/19-client-server-cli.md

@@ -42,7 +42,6 @@ This creates several limitations:
    - Add new endpoints to handle CLI build requests
    - Move analysis and execution logic from CLI to service
    - Service maintains orchestrator state and coordinates builds
-   - Add real-time progress streaming for CLI consumption
 
 2. **Add CLI-Specific API Endpoints**
    - `/api/v1/cli/build` - Handle build requests from CLI