databuild/databuild/databuild.proto

syntax = "proto3";

package databuild.v1;

message PartitionRef {
  string str = 1;
}

///////////////////////////////////////////////////////////////////////////////////////////////
// Jobs
///////////////////////////////////////////////////////////////////////////////////////////////

//
// Job Config
//

// The type of dependency
enum DepType {
  QUERY = 0; // Default
  MATERIALIZE = 1;
}

// Represents a data dependency
message DataDep {
  DepType dep_type = 1;
  PartitionRef partition_ref = 2;
}

// Configuration for a job
message JobConfig {
  // The partitions that this parameterization produces
  repeated PartitionRef outputs = 1;

  // Required data dependencies
  repeated DataDep inputs = 2;

  // Command line arguments
  repeated string args = 3;

  // Environment variables
  map<string, string> env = 4;
}

// Request message for job configuration service
message JobConfigureRequest { repeated PartitionRef outputs = 1; }

// Response message for job configuration service
message JobConfigureResponse { repeated JobConfig configs = 1; }

// Implemented by the job.cfg bazel rule
service JobConfigure {
  rpc Configure(JobConfigureRequest) returns (JobConfigureResponse);
}

//
// Job Exec
//

// Manifest that records the literal partitions consumed (and their manifests) in order to
// produce the specified partitions
message PartitionManifest {
  // The refs of the partitions produced by this job
  repeated PartitionRef outputs = 1;

  // Input partition manifests
  repeated PartitionManifest inputs = 2;

  // Start time of job execution (Unix timestamp seconds)
  int64 start_time = 3;

  // End time of job execution (Unix timestamp seconds)
  int64 end_time = 4;

  // The configuration used to run the job
  Task task = 5;
}

message JobExecuteRequest { repeated PartitionRef outputs = 1; }

// Metadata for the complete set of partitions produced by this job
message JobExecuteResponse { repeated PartitionManifest manifests = 1; }

// Implemented by the job.exec bazel rule
service JobExecute {
  rpc Execute(JobExecuteRequest) returns (JobExecuteResponse);
}

///////////////////////////////////////////////////////////////////////////////////////////////
// Graphs
///////////////////////////////////////////////////////////////////////////////////////////////

//
// GraphLookup
//

message JobLabel {
  // The bazel label the references the job_target
  string label = 1;
}

message GraphLookupRequest { repeated PartitionRef outputs = 1; }

// Represents a not-yet configured task
message TaskRef {
  // The job whose configure/exec targets will be used
  JobLabel job = 1;

  // The partition refs this task is responsible for producing, and with which the configure
  // target will be invoked
  repeated PartitionRef outputs = 2;
}

// Represents the complete set of tasks needed to produce the requested partitions
message GraphLookupResponse { repeated TaskRef task_refs = 1; }

// Implemented per graph
service GraphLookup {
  rpc Lookup(GraphLookupRequest) returns (GraphLookupResponse);
}

// Request message for graph analyze service
message GraphAnalyzeRequest { repeated PartitionRef outputs = 1; }

//
// JobGraph
//

message Task {
  // The bazel label uniquely identifying the job
  JobLabel job = 1;

  // The configuration for the job
  JobConfig config = 2;
}

// The bazel label referencing the graph
message GraphLabel { string label = 1; }

// Represents a job graph
message JobGraph {
  // The bazel label of the graph to be executed
  GraphLabel label = 1;

  // The output partitions to be produced by this graph
  repeated PartitionRef outputs = 2;

  // The job configurations that make up this graph
  repeated Task nodes = 3;
}

// Response message for graph analyze service
message GraphAnalyzeResponse { JobGraph graph = 1; }

message GraphExecuteResponse { repeated PartitionManifest manifests = 1; }
message GraphBuildRequest { repeated PartitionRef outputs = 1; }
message GraphBuildResponse { repeated PartitionManifest manifests = 1; }

///////////////////////////////////////////////////////////////////////////////////////////////
// Build Event Log
///////////////////////////////////////////////////////////////////////////////////////////////

// Partition lifecycle states
enum PartitionStatus {
  PARTITION_UNKNOWN = 0;
  PARTITION_REQUESTED = 1;     // Partition requested but not yet analyzed
  PARTITION_ANALYZED = 2;      // Partition analyzed successfully
  PARTITION_BUILDING = 3;      // Job actively building this partition
  PARTITION_AVAILABLE = 4;     // Partition successfully built and available
  PARTITION_FAILED = 5;        // Partition build failed
  PARTITION_DELEGATED = 6;     // Request delegated to existing build
}

// Job execution lifecycle
enum JobStatus {
  JOB_UNKNOWN = 0;
  JOB_SCHEDULED = 1;           // Job scheduled for execution
  JOB_RUNNING = 2;             // Job actively executing
  JOB_COMPLETED = 3;           // Job completed successfully
  JOB_FAILED = 4;              // Job execution failed
  JOB_CANCELLED = 5;           // Job execution cancelled
  JOB_SKIPPED = 6;             // Job skipped because target partitions already available
}

// Build request lifecycle
enum BuildRequestStatus {
  BUILD_REQUEST_UNKNOWN = 0;
  BUILD_REQUEST_RECEIVED = 1;         // Build request received and queued
  BUILD_REQUEST_PLANNING = 2;         // Graph analysis in progress
  BUILD_REQUEST_ANALYSIS_COMPLETED = 7; // Graph analysis completed successfully
  BUILD_REQUEST_EXECUTING = 3;        // Jobs are being executed
  BUILD_REQUEST_COMPLETED = 4;        // All requested partitions built
  BUILD_REQUEST_FAILED = 5;           // Build request failed
  BUILD_REQUEST_CANCELLED = 6;        // Build request cancelled
}

// Build request lifecycle event
message BuildRequestEvent {
  BuildRequestStatus status = 1;
  repeated PartitionRef requested_partitions = 2;
  string message = 3;                     // Optional status message
}

// Partition state change event
message PartitionEvent {
  PartitionRef partition_ref = 1;
  PartitionStatus status = 2;
  string message = 3;                     // Optional status message
  string job_run_id = 4;                  // UUID of job run producing this partition (if applicable)
}

// Job execution event
message JobEvent {
  string job_run_id = 1;                  // UUID for this job run
  JobLabel job_label = 2;                 // Job being executed
  repeated PartitionRef target_partitions = 3; // Partitions this job run produces
  JobStatus status = 4;
  string message = 5;                     // Optional status message
  JobConfig config = 6;                   // Job configuration used (for SCHEDULED events)
  repeated PartitionManifest manifests = 7; // Results (for COMPLETED events)
}

// Delegation event (when build request delegates to existing build)
message DelegationEvent {
  PartitionRef partition_ref = 1;
  string delegated_to_build_request_id = 2; // Build request handling this partition
  string message = 3;                     // Optional message
}

// Job graph analysis result event (stores the analyzed job graph)
message JobGraphEvent {
  JobGraph job_graph = 1;                 // The analyzed job graph
  string message = 2;                     // Optional message
}

// Individual build event
message BuildEvent {
  // Event metadata
  string event_id = 1;                    // UUID for this event
  int64 timestamp = 2;                    // Unix timestamp (nanoseconds)
  string build_request_id = 3;            // UUID of the build request
  
  // Event type and payload (one of)
  oneof event_type {
    BuildRequestEvent build_request_event = 10;
    PartitionEvent partition_event = 11;
    JobEvent job_event = 12;
    DelegationEvent delegation_event = 13;
    JobGraphEvent job_graph_event = 14;
  }
}

///////////////////////////////////////////////////////////////////////////////////////////////
// Services
///////////////////////////////////////////////////////////////////////////////////////////////

// Service for job configuration and graph analysis
service DataBuildService {
//  // Get job configurations for the specified output references
//  rpc GetJobConfigs(JobConfigureRequest) returns (JobConfigureResponse) {}

  // Analyze and get the job graph for the specified output references
  rpc AnalyzeGraph(GraphAnalyzeRequest) returns (GraphAnalyzeResponse);

  // Execute the specified job graph (implemented by databuild)
  rpc Execute(JobGraph) returns (GraphExecuteResponse);

  // User-facing: build the desired partitions
  rpc Build(GraphBuildRequest) returns (GraphBuildResponse);
}