databuild/databuild/databuild.proto

syntax = "proto3";

package databuild.v1;

message PartitionRef {
  string str = 1;
}

///////////////////////////////////////////////////////////////////////////////////////////////
// Jobs
///////////////////////////////////////////////////////////////////////////////////////////////

//
// Job Config
//

// The type of dependency
enum DepType {
  QUERY = 0; // Default
  MATERIALIZE = 1;
}

// Represents a data dependency
message DataDep {
  DepType dep_type = 1;
  PartitionRef partition_ref = 2;
}

// Configuration for a job
message JobConfig {
  // The partitions that this parameterization produces
  repeated PartitionRef outputs = 1;

  // Required data dependencies
  repeated DataDep inputs = 2;

  // Command line arguments
  repeated string args = 3;

  // Environment variables
  map<string, string> env = 4;
}

// Request message for job configuration service
message JobConfigureRequest { repeated PartitionRef outputs = 1; }

// Response message for job configuration service
message JobConfigureResponse { repeated JobConfig configs = 1; }

// Implemented by the job.cfg bazel rule
service JobConfigure {
  rpc Configure(JobConfigureRequest) returns (JobConfigureResponse);
}

//
// Job Exec
//

// Manifest that records the literal partitions consumed (and their manifests) in order to
// produce the specified partitions
message PartitionManifest {
  // The refs of the partitions produced by this job
  repeated PartitionRef outputs = 1;

  // Input partition manifests
  repeated PartitionManifest inputs = 2;

  // Start time of job execution (Unix timestamp seconds)
  int64 start_time = 3;

  // End time of job execution (Unix timestamp seconds)
  int64 end_time = 4;

  // The configuration used to run the job
  Task task = 5;
}

message JobExecuteRequest { repeated PartitionRef outputs = 1; }

// Metadata for the complete set of partitions produced by this job
message JobExecuteResponse { repeated PartitionManifest manifests = 1; }

// Implemented by the job.exec bazel rule
service JobExecute {
  rpc Execute(JobExecuteRequest) returns (JobExecuteResponse);
}

///////////////////////////////////////////////////////////////////////////////////////////////
// Graphs
///////////////////////////////////////////////////////////////////////////////////////////////

//
// GraphLookup
//

message JobLabel {
  // The bazel label the references the job_target
  string label = 1;
}

message GraphLookupRequest { repeated PartitionRef outputs = 1; }

// Represents a not-yet configured task
message TaskRef {
  // The job whose configure/exec targets will be used
  JobLabel job = 1;

  // The partition refs this task is responsible for producing, and with which the configure
  // target will be invoked
  repeated PartitionRef outputs = 2;
}

// Represents the complete set of tasks needed to produce the requested partitions
message GraphLookupResponse { repeated TaskRef task_refs = 1; }

// Implemented per graph
service GraphLookup {
  rpc Lookup(GraphLookupRequest) returns (GraphLookupResponse);
}

// Request message for graph analyze service
message GraphAnalyzeRequest { repeated PartitionRef outputs = 1; }

//
// JobGraph
//

message Task {
  // The bazel label uniquely identifying the job
  JobLabel job = 1;

  // The configuration for the job
  JobConfig config = 2;
}

// The bazel label referencing the graph
message GraphLabel { string label = 1; }

// Represents a job graph
message JobGraph {
  // The bazel label of the graph to be executed
  GraphLabel label = 1;

  // The output partitions to be produced by this graph
  repeated PartitionRef outputs = 2;

  // The job configurations that make up this graph
  repeated Task nodes = 3;
}

// Response message for graph analyze service
message GraphAnalyzeResponse { JobGraph graph = 1; }

message GraphExecuteResponse { repeated PartitionManifest manifests = 1; }
message GraphBuildRequest { repeated PartitionRef outputs = 1; }
message GraphBuildResponse { repeated PartitionManifest manifests = 1; }

// Service for job configuration and graph analysis
service DataBuildService {
//  // Get job configurations for the specified output references
//  rpc GetJobConfigs(JobConfigureRequest) returns (JobConfigureResponse) {}

  // Analyze and get the job graph for the specified output references
  rpc AnalyzeGraph(GraphAnalyzeRequest) returns (GraphAnalyzeResponse);

  // Execute the specified job graph (implemented by databuild)
  rpc Execute(JobGraph) returns (GraphExecuteResponse);

  // User-facing: build the desired partitions
  rpc Build(GraphBuildRequest) returns (GraphBuildResponse);
}