diff --git a/.bazelrc b/.bazelrc new file mode 100644 index 0000000..3912897 --- /dev/null +++ b/.bazelrc @@ -0,0 +1,4 @@ +build --color=yes +query --color=yes +test --color=yes +test --test_output=errors \ No newline at end of file diff --git a/.bazelversion b/.bazelversion index 88dee36..797ed2e 100644 --- a/.bazelversion +++ b/.bazelversion @@ -1 +1 @@ -8.2.0 \ No newline at end of file +8.2.1 \ No newline at end of file diff --git a/.envrc b/.envrc new file mode 100644 index 0000000..30b7f9c --- /dev/null +++ b/.envrc @@ -0,0 +1,2 @@ + +PATH_add scripts/ diff --git a/.gitignore b/.gitignore index f646a92..a51dcfb 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,4 @@ databuild.iml .idea .DS_Store examples/podcast_reviews/data +.bazelbsp diff --git a/BUILD.bazel b/BUILD.bazel index 33e8aef..2bc01e1 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -6,7 +6,7 @@ filegroup( ) filegroup( - name = "json_schema", - srcs = ["databuild.schema.json"], + name = "proto", + srcs = ["databuild.proto"], visibility = ["//visibility:public"], ) diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock index 0b62bb8..9ba5c40 100644 --- a/MODULE.bazel.lock +++ b/MODULE.bazel.lock @@ -657,7 +657,7 @@ }, "@@rules_go+//go:extensions.bzl%go_sdk": { "os:osx,arch:aarch64": { - "bzlTransitiveDigest": "rxbwsDnZCD2rggzpxiw2BVDOKaZyZqrTcwmg4Zhsecc=", + "bzlTransitiveDigest": "jBP0cRKOr+A42aPGunoasOD+vrmMLJIJ8Jwi65DdelE=", "usagesDigest": "9WUdtwMNxQMIp54gOxEBVws3WnIUdQcvX9pRfBtrtvQ=", "recordedFileInputs": {}, "recordedDirentsInputs": {}, diff --git a/databuild.proto b/databuild.proto new file mode 100644 index 0000000..2574931 --- /dev/null +++ b/databuild.proto @@ -0,0 +1,100 @@ +edition = "2023"; + +import "google/protobuf/timestamp.proto"; + +package databuild.v1; + +// The type of dependency +enum DepType { + QUERY = 0; // Default + MATERIALIZE = 1; +} + +// Represents a data dependency +message DataDep { + DepType dep_type = 1; + string partition_ref = 2; +} + +// Configuration for a job +message JobConfig { + // The partitions that this parameterization produces + repeated string outputs = 1; + + // Required data dependencies + repeated DataDep inputs = 2; + + // Command line arguments + repeated string args = 3; + + // Environment variables + map env = 4; +} + +// Represents a partition manifest +message PartitionManifest { + // The refs of the partitions produced by this job + repeated string outputs = 1; + + // Input partition manifests + repeated PartitionManifest inputs = 2; + + // Start time of job execution (Unix timestamp) + google.protobuf.Timestamp start_time = 3; + + // End time of job execution (Unix timestamp) + google.protobuf.Timestamp end_time = 4; + + // The configuration used to run the job + Task task = 5; +} + +message Task { + // The bazel label uniquely identifying the job + string job_label = 1; + + // The configuration for the job + JobConfig config = 2; +} + +// Represents a job graph +message JobGraph { + // The output partitions to be produced by this graph + repeated string outputs = 1; + + // The job configurations that make up this graph + repeated Task nodes = 2; +} + +// Request message for job configuration service +message JobConfigRequest { + // The output references to configure jobs for + repeated string output_refs = 1; +} + +// Response message for job configuration service +message JobConfigResponse { + // The job configurations + repeated JobConfig configs = 1; +} + +// Request message for graph analyze service +message GraphAnalyzeRequest { + // The output references to analyze + repeated string output_refs = 1; +} + +// Response message for graph analyze service +message GraphAnalyzeResponse { + // The job graph + JobGraph graph = 1; +} + +// Service for job configuration and graph analysis +service DataBuildService { + // Get job configurations for the specified output references + rpc GetJobConfigs(JobConfigRequest) returns (JobConfigResponse) {} + + // Analyze and get the job graph for the specified output references + rpc AnalyzeGraph(GraphAnalyzeRequest) returns (GraphAnalyzeResponse) {} +} diff --git a/databuild.schema.json b/databuild.schema.json deleted file mode 100644 index a9e3bfc..0000000 --- a/databuild.schema.json +++ /dev/null @@ -1,102 +0,0 @@ -{ - "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "https://databuild.example.com/schemas/v1", - "title": "DataBuild Core Schemas", - "description": "Schema definitions for DataBuild core components", - - "definitions": { - "DepType": { - "type": "string", - "enum": ["query", "materialize"], - "description": "The type of dependency" - }, - - "DataDep": { - "type": "object", - "required": ["depType", "ref"], - "properties": { - "depType": { - "$ref": "#/definitions/DepType" - }, - "ref": { - "type": "string" - } - } - }, - - "JobConfig": { - "type": "object", - "required": ["outputs", "inputs","args","env"], - "properties": { - "outputs": { - "type": "array", - "items": { "type": "string" }, - "description": "The partitions that this parameterization produces" - }, - "inputs": { - "type": "array", - "items": { "$ref": "#/definitions/DataDep" }, - "description": "Required data dependencies" - }, - "args": { - "type": "array", - "items": { "type": "string" }, - "description": "Command line arguments" - }, - "env": { - "type": "object", - "additionalProperties": { "type": "string" }, - "description": "Environment variables" - } - } - }, - - "PartitionManifest": { - "type": "object", - "required": ["outputs", "inputs", "startTime", "endTime", "config"], - "properties": { - "outputs": { - "type": "array", - "items": { "type": "string" }, - "description": "The refs of the partitions produced by this job" - }, - "inputs": { - "type": "array", - "items": { "$ref": "#/definitions/PartitionManifest" }, - "description": "Input partition manifests" - }, - "startTime": { - "type": "integer", - "format": "int64", - "description": "Start time of job execution (Unix timestamp)" - }, - "endTime": { - "type": "integer", - "format": "int64", - "description": "End time of job execution (Unix timestamp)" - }, - "config": { - "$ref": "#/definitions/JobConfig", - "description": "The configuration used to run the job" - } - } - }, - - "JobGraph": { - "type": "object", - "required": ["outputs", "nodes"], - "properties": { - "outputs": { - "type": "array", - "items": { "type": "string" }, - "description": "The output partitions to be produced by this graph" - }, - "nodes": { - "type": "array", - "items": { "$ref": "#/definitions/JobConfig" }, - "description": "The job configurations that make up this graph" - } - } - } - } -} \ No newline at end of file diff --git a/examples/basic_graph/GenerateExecute.java b/examples/basic_graph/GenerateExecute.java index 0e4064b..671d727 100644 --- a/examples/basic_graph/GenerateExecute.java +++ b/examples/basic_graph/GenerateExecute.java @@ -12,7 +12,7 @@ import java.util.Random; * This class generates a random number based on the partition ref. */ public class GenerateExecute { - public static String BASE_PATH = "/tmp/databuild/examples/basic_graph/"; + public static String BASE_PATH = "/tmp/databuild_test/examples/basic_graph/"; public static void main(String[] args) { if (args.length < 1) { diff --git a/examples/basic_graph/README.md b/examples/basic_graph/README.md index b331440..fc4e59b 100644 --- a/examples/basic_graph/README.md +++ b/examples/basic_graph/README.md @@ -4,7 +4,7 @@ This example demonstrates a databuild_job that generates a random number seeded ## Multiple Configs -We can generate numbers for any partition provided (written to `/tmp/databuild/examples/basic_graph`), and so we have +We can generate numbers for any partition provided (written to `/tmp/databuild_test/examples/basic_graph`), and so we have a config per partition for demonstration purposes: ```bash diff --git a/examples/basic_graph/test/exec_test.sh b/examples/basic_graph/test/exec_test.sh index 3d6b4ba..8045c8d 100644 --- a/examples/basic_graph/test/exec_test.sh +++ b/examples/basic_graph/test/exec_test.sh @@ -2,7 +2,7 @@ set -e # Test the .exec rule -basic_graph.exec < <(basic_graph.analyze /tmp/databuild/examples/basic_graph/generated_number/pippin_salem_sadie) +basic_graph.exec < <(basic_graph.analyze /tmp/databuild_test/examples/basic_graph/generated_number/pippin_salem_sadie) # Test the .build rule -basic_graph.build /tmp/databuild/examples/basic_graph/generated_number/pippin_salem_sadie \ No newline at end of file +basic_graph.build /tmp/databuild_test/examples/basic_graph/generated_number/pippin_salem_sadie \ No newline at end of file diff --git a/examples/basic_graph/test/generate_number_test.sh b/examples/basic_graph/test/generate_number_test.sh index 6b8c00d..9225dae 100755 --- a/examples/basic_graph/test/generate_number_test.sh +++ b/examples/basic_graph/test/generate_number_test.sh @@ -2,14 +2,14 @@ set -e # Test configure -generate_number_job.cfg /tmp/databuild/examples/basic_graph/generated_number/pippin /tmp/databuild/examples/basic_graph/generated_number/salem /tmp/databuild/examples/basic_graph/generated_number/sadie +generate_number_job.cfg /tmp/databuild_test/examples/basic_graph/generated_number/pippin /tmp/databuild_test/examples/basic_graph/generated_number/salem /tmp/databuild_test/examples/basic_graph/generated_number/sadie # Test run -generate_number_job.cfg /tmp/databuild/examples/basic_graph/generated_number/pippin | jq -c ".[0]" | generate_number_job.exec +generate_number_job.cfg /tmp/databuild_test/examples/basic_graph/generated_number/pippin | jq -c ".[0]" | generate_number_job.exec -# Validate that contents of pippin is 57 -if [[ "$(cat /tmp/databuild/examples/basic_graph/generated_number/pippin)" != "57" ]]; then - echo "Assertion failed: File does not contain 57" - cat /tmp/databuild/examples/basic_graph/generated_number/pippin +# Validate that contents of pippin is 83 +if [[ "$(cat /tmp/databuild_test/examples/basic_graph/generated_number/pippin)" != "83" ]]; then + echo "Assertion failed: File does not contain 83" + cat /tmp/databuild_test/examples/basic_graph/generated_number/pippin exit 1 fi diff --git a/examples/basic_graph/test/graph_test.sh b/examples/basic_graph/test/graph_test.sh index 221cd54..5942bba 100755 --- a/examples/basic_graph/test/graph_test.sh +++ b/examples/basic_graph/test/graph_test.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash set -e -basic_graph.lookup /tmp/databuild/examples/basic_graph/generated_number/pippin_salem_sadie +basic_graph.lookup /tmp/databuild_test/examples/basic_graph/generated_number/pippin_salem_sadie -basic_graph.analyze /tmp/databuild/examples/basic_graph/generated_number/pippin_salem_sadie +basic_graph.analyze /tmp/databuild_test/examples/basic_graph/generated_number/pippin_salem_sadie diff --git a/examples/basic_graph/test/sum_test.sh b/examples/basic_graph/test/sum_test.sh index 23628c2..d4f3490 100755 --- a/examples/basic_graph/test/sum_test.sh +++ b/examples/basic_graph/test/sum_test.sh @@ -5,15 +5,15 @@ set -e sum_job.cfg pippin_salem_sadie # Test run -echo -n 57 > /tmp/databuild/examples/basic_graph/generated_number/pippin -echo -n 59 > /tmp/databuild/examples/basic_graph/generated_number/salem -echo -n 1 > /tmp/databuild/examples/basic_graph/generated_number/sadie +echo -n 83 > /tmp/databuild_test/examples/basic_graph/generated_number/pippin +echo -n 34 > /tmp/databuild_test/examples/basic_graph/generated_number/salem +echo -n 19 > /tmp/databuild_test/examples/basic_graph/generated_number/sadie sum_job.cfg pippin_salem_sadie | jq -c ".[0]" | sum_job.exec -# Validate that contents of pippin is 43 -if [[ "$(cat /tmp/databuild/examples/basic_graph/sum/pippin_salem_sadie)" != "117" ]]; then - echo "Assertion failed: File does not contain 117" - cat /tmp/databuild/examples/basic_graph/sum/pippin_salem_sadie +# Validate that contents of output is 136 +if [[ "$(cat /tmp/databuild_test/examples/basic_graph/sum/pippin_salem_sadie)" != "136" ]]; then + echo "Assertion failed: File does not contain 136" + cat /tmp/databuild_test/examples/basic_graph/sum/pippin_salem_sadie exit 1 fi diff --git a/examples/hello_world/configure.sh b/examples/hello_world/configure.sh index 342c366..d69fa5b 100755 --- a/examples/hello_world/configure.sh +++ b/examples/hello_world/configure.sh @@ -1,2 +1,2 @@ # Create a test job config -echo "[{\"outputs\":[\"$1\"],\"inputs\":[],\"args\":[\"will\", \"build\", \"$1\"],\"env\":{\"foo\":\"bar\"}}]" +echo "{\"configs\":[{\"outputs\":[\"$1\"],\"inputs\":[],\"args\":[\"will\", \"build\", \"$1\"],\"env\":{\"foo\":\"bar\"}}]}" diff --git a/examples/hello_world/test/test.sh b/examples/hello_world/test/test.sh index 7b98166..c90726f 100755 --- a/examples/hello_world/test/test.sh +++ b/examples/hello_world/test/test.sh @@ -2,4 +2,4 @@ test_job.cfg nice -test_job.cfg cool | jq -c ".[0]" | test_job.exec +test_job.cfg cool | jq -c ".configs[0]" | test_job.exec diff --git a/examples/podcast_reviews/MODULE.bazel b/examples/podcast_reviews/MODULE.bazel index 6caa4dd..69ccd92 100644 --- a/examples/podcast_reviews/MODULE.bazel +++ b/examples/podcast_reviews/MODULE.bazel @@ -1,5 +1,5 @@ module( - name = "databuild_app_reviews", + name = "databuild_podcast_reviews", version = "0.1", ) diff --git a/examples/podcast_reviews/MODULE.bazel.lock b/examples/podcast_reviews/MODULE.bazel.lock index 14bb922..3117aab 100644 --- a/examples/podcast_reviews/MODULE.bazel.lock +++ b/examples/podcast_reviews/MODULE.bazel.lock @@ -10,8 +10,12 @@ "https://bcr.bazel.build/modules/abseil-cpp/20230802.1/MODULE.bazel": "fa92e2eb41a04df73cdabeec37107316f7e5272650f81d6cc096418fe647b915", "https://bcr.bazel.build/modules/abseil-cpp/20240116.1/MODULE.bazel": "37bcdb4440fbb61df6a1c296ae01b327f19e9bb521f9b8e26ec854b6f97309ed", "https://bcr.bazel.build/modules/abseil-cpp/20240116.1/source.json": "9be551b8d4e3ef76875c0d744b5d6a504a27e3ae67bc6b28f46415fd2d2957da", + "https://bcr.bazel.build/modules/aspect_bazel_lib/2.14.0/MODULE.bazel": "2b31ffcc9bdc8295b2167e07a757dbbc9ac8906e7028e5170a3708cecaac119f", + "https://bcr.bazel.build/modules/aspect_bazel_lib/2.14.0/source.json": "0cf1826853b0bef8b5cd19c0610d717500f5521aa2b38b72b2ec302ac5e7526c", + "https://bcr.bazel.build/modules/aspect_bazel_lib/2.7.2/MODULE.bazel": "780d1a6522b28f5edb7ea09630748720721dfe27690d65a2d33aa7509de77e07", "https://bcr.bazel.build/modules/bazel_features/1.1.0/MODULE.bazel": "cfd42ff3b815a5f39554d97182657f8c4b9719568eb7fded2b9135f084bf760b", "https://bcr.bazel.build/modules/bazel_features/1.1.1/MODULE.bazel": "27b8c79ef57efe08efccbd9dd6ef70d61b4798320b8d3c134fd571f78963dbcd", + "https://bcr.bazel.build/modules/bazel_features/1.10.0/MODULE.bazel": "f75e8807570484a99be90abcd52b5e1f390362c258bcb73106f4544957a48101", "https://bcr.bazel.build/modules/bazel_features/1.11.0/MODULE.bazel": "f9382337dd5a474c3b7d334c2f83e50b6eaedc284253334cf823044a26de03e8", "https://bcr.bazel.build/modules/bazel_features/1.15.0/MODULE.bazel": "d38ff6e517149dc509406aca0db3ad1efdd890a85e049585b7234d04238e2a4d", "https://bcr.bazel.build/modules/bazel_features/1.17.0/MODULE.bazel": "039de32d21b816b47bd42c778e0454217e9c9caac4a3cf8e15c7231ee3ddee4d", @@ -20,6 +24,7 @@ "https://bcr.bazel.build/modules/bazel_features/1.21.0/MODULE.bazel": "675642261665d8eea09989aa3b8afb5c37627f1be178382c320d1b46afba5e3b", "https://bcr.bazel.build/modules/bazel_features/1.21.0/source.json": "3e8379efaaef53ce35b7b8ba419df829315a880cb0a030e5bb45c96d6d5ecb5f", "https://bcr.bazel.build/modules/bazel_features/1.4.1/MODULE.bazel": "e45b6bb2350aff3e442ae1111c555e27eac1d915e77775f6fdc4b351b758b5d7", + "https://bcr.bazel.build/modules/bazel_features/1.9.0/MODULE.bazel": "885151d58d90d8d9c811eb75e3288c11f850e1d6b481a8c9f766adee4712358b", "https://bcr.bazel.build/modules/bazel_features/1.9.1/MODULE.bazel": "8f679097876a9b609ad1f60249c49d68bfab783dd9be012faf9d82547b14815a", "https://bcr.bazel.build/modules/bazel_skylib/1.0.3/MODULE.bazel": "bcb0fd896384802d1ad283b4e4eb4d718eebd8cb820b0a2c3a347fb971afd9d8", "https://bcr.bazel.build/modules/bazel_skylib/1.1.1/MODULE.bazel": "1add3e7d93ff2e6998f9e118022c84d163917d912f5afafb3058e3d2f1545b5e", @@ -93,6 +98,7 @@ "https://bcr.bazel.build/modules/rules_java/4.0.0/MODULE.bazel": "5a78a7ae82cd1a33cef56dc578c7d2a46ed0dca12643ee45edbb8417899e6f74", "https://bcr.bazel.build/modules/rules_java/5.3.5/MODULE.bazel": "a4ec4f2db570171e3e5eb753276ee4b389bae16b96207e9d3230895c99644b86", "https://bcr.bazel.build/modules/rules_java/6.0.0/MODULE.bazel": "8a43b7df601a7ec1af61d79345c17b31ea1fedc6711fd4abfd013ea612978e39", + "https://bcr.bazel.build/modules/rules_java/6.3.0/MODULE.bazel": "a97c7678c19f236a956ad260d59c86e10a463badb7eb2eda787490f4c969b963", "https://bcr.bazel.build/modules/rules_java/6.4.0/MODULE.bazel": "e986a9fe25aeaa84ac17ca093ef13a4637f6107375f64667a15999f77db6c8f6", "https://bcr.bazel.build/modules/rules_java/6.5.2/MODULE.bazel": "1d440d262d0e08453fa0c4d8f699ba81609ed0e9a9a0f02cd10b3e7942e61e31", "https://bcr.bazel.build/modules/rules_java/7.10.0/MODULE.bazel": "530c3beb3067e870561739f1144329a21c851ff771cd752a49e06e3dc9c2e71a", @@ -118,6 +124,8 @@ "https://bcr.bazel.build/modules/rules_license/0.0.7/MODULE.bazel": "088fbeb0b6a419005b89cf93fe62d9517c0a2b8bb56af3244af65ecfe37e7d5d", "https://bcr.bazel.build/modules/rules_license/1.0.0/MODULE.bazel": "a7fda60eefdf3d8c827262ba499957e4df06f659330bbe6cdbdb975b768bb65c", "https://bcr.bazel.build/modules/rules_license/1.0.0/source.json": "a52c89e54cc311196e478f8382df91c15f7a2bfdf4c6cd0e2675cc2ff0b56efb", + "https://bcr.bazel.build/modules/rules_oci/2.2.6/MODULE.bazel": "2ba6ddd679269e00aeffe9ca04faa2d0ca4129650982c9246d0d459fe2da47d9", + "https://bcr.bazel.build/modules/rules_oci/2.2.6/source.json": "94e7decb8f95d9465b0bbea71c65064cd16083be1350c7468f131818641dc4a5", "https://bcr.bazel.build/modules/rules_pkg/0.7.0/MODULE.bazel": "df99f03fc7934a4737122518bb87e667e62d780b610910f0447665a7e2be62dc", "https://bcr.bazel.build/modules/rules_pkg/1.0.1/MODULE.bazel": "5b1df97dbc29623bccdf2b0dcd0f5cb08e2f2c9050aab1092fd39a41e82686ff", "https://bcr.bazel.build/modules/rules_pkg/1.0.1/source.json": "bd82e5d7b9ce2d31e380dd9f50c111d678c3bdaca190cb76b0e1c71b05e1ba8a", @@ -140,7 +148,9 @@ "https://bcr.bazel.build/modules/rules_shell/0.4.0/source.json": "1d7fa7f941cd41dc2704ba5b4edc2e2230eea1cc600d80bd2b65838204c50b95", "https://bcr.bazel.build/modules/stardoc/0.5.1/MODULE.bazel": "1a05d92974d0c122f5ccf09291442580317cdd859f07a8655f1db9a60374f9f8", "https://bcr.bazel.build/modules/stardoc/0.5.3/MODULE.bazel": "c7f6948dae6999bf0db32c1858ae345f112cacf98f174c7a8bb707e41b974f1c", + "https://bcr.bazel.build/modules/stardoc/0.5.4/MODULE.bazel": "6569966df04610b8520957cb8e97cf2e9faac2c0309657c537ab51c16c18a2a4", "https://bcr.bazel.build/modules/stardoc/0.5.6/MODULE.bazel": "c43dabc564990eeab55e25ed61c07a1aadafe9ece96a4efabb3f8bf9063b71ef", + "https://bcr.bazel.build/modules/stardoc/0.6.2/MODULE.bazel": "7060193196395f5dd668eda046ccbeacebfd98efc77fed418dbe2b82ffaa39fd", "https://bcr.bazel.build/modules/stardoc/0.7.0/MODULE.bazel": "05e3d6d30c099b6770e97da986c53bd31844d7f13d41412480ea265ac9e8079c", "https://bcr.bazel.build/modules/stardoc/0.7.1/MODULE.bazel": "3548faea4ee5dda5580f9af150e79d0f6aea934fc60c1cc50f4efdd9420759e7", "https://bcr.bazel.build/modules/stardoc/0.7.2/MODULE.bazel": "fc152419aa2ea0f51c29583fab1e8c99ddefd5b3778421845606ee628629e0e5", @@ -401,10 +411,167 @@ ] } }, + "@@rules_oci+//oci:extensions.bzl%oci": { + "general": { + "bzlTransitiveDigest": "8fB61KHYOU4XHH65DqVw59ZIDGO29I2WIbVHxii4slA=", + "usagesDigest": "/O1PwnnkqSBmI9Oe08ZYYqjM4IS8JR+/9rjgzVTNDaQ=", + "recordedFileInputs": {}, + "recordedDirentsInputs": {}, + "envVariables": {}, + "generatedRepoSpecs": { + "oci_crane_darwin_amd64": { + "repoRuleId": "@@rules_oci+//oci:repositories.bzl%crane_repositories", + "attributes": { + "platform": "darwin_amd64", + "crane_version": "v0.18.0" + } + }, + "oci_crane_darwin_arm64": { + "repoRuleId": "@@rules_oci+//oci:repositories.bzl%crane_repositories", + "attributes": { + "platform": "darwin_arm64", + "crane_version": "v0.18.0" + } + }, + "oci_crane_linux_arm64": { + "repoRuleId": "@@rules_oci+//oci:repositories.bzl%crane_repositories", + "attributes": { + "platform": "linux_arm64", + "crane_version": "v0.18.0" + } + }, + "oci_crane_linux_armv6": { + "repoRuleId": "@@rules_oci+//oci:repositories.bzl%crane_repositories", + "attributes": { + "platform": "linux_armv6", + "crane_version": "v0.18.0" + } + }, + "oci_crane_linux_i386": { + "repoRuleId": "@@rules_oci+//oci:repositories.bzl%crane_repositories", + "attributes": { + "platform": "linux_i386", + "crane_version": "v0.18.0" + } + }, + "oci_crane_linux_s390x": { + "repoRuleId": "@@rules_oci+//oci:repositories.bzl%crane_repositories", + "attributes": { + "platform": "linux_s390x", + "crane_version": "v0.18.0" + } + }, + "oci_crane_linux_amd64": { + "repoRuleId": "@@rules_oci+//oci:repositories.bzl%crane_repositories", + "attributes": { + "platform": "linux_amd64", + "crane_version": "v0.18.0" + } + }, + "oci_crane_windows_armv6": { + "repoRuleId": "@@rules_oci+//oci:repositories.bzl%crane_repositories", + "attributes": { + "platform": "windows_armv6", + "crane_version": "v0.18.0" + } + }, + "oci_crane_windows_amd64": { + "repoRuleId": "@@rules_oci+//oci:repositories.bzl%crane_repositories", + "attributes": { + "platform": "windows_amd64", + "crane_version": "v0.18.0" + } + }, + "oci_crane_toolchains": { + "repoRuleId": "@@rules_oci+//oci/private:toolchains_repo.bzl%toolchains_repo", + "attributes": { + "toolchain_type": "@rules_oci//oci:crane_toolchain_type", + "toolchain": "@oci_crane_{platform}//:crane_toolchain" + } + }, + "oci_regctl_darwin_amd64": { + "repoRuleId": "@@rules_oci+//oci:repositories.bzl%regctl_repositories", + "attributes": { + "platform": "darwin_amd64" + } + }, + "oci_regctl_darwin_arm64": { + "repoRuleId": "@@rules_oci+//oci:repositories.bzl%regctl_repositories", + "attributes": { + "platform": "darwin_arm64" + } + }, + "oci_regctl_linux_arm64": { + "repoRuleId": "@@rules_oci+//oci:repositories.bzl%regctl_repositories", + "attributes": { + "platform": "linux_arm64" + } + }, + "oci_regctl_linux_s390x": { + "repoRuleId": "@@rules_oci+//oci:repositories.bzl%regctl_repositories", + "attributes": { + "platform": "linux_s390x" + } + }, + "oci_regctl_linux_amd64": { + "repoRuleId": "@@rules_oci+//oci:repositories.bzl%regctl_repositories", + "attributes": { + "platform": "linux_amd64" + } + }, + "oci_regctl_windows_amd64": { + "repoRuleId": "@@rules_oci+//oci:repositories.bzl%regctl_repositories", + "attributes": { + "platform": "windows_amd64" + } + }, + "oci_regctl_toolchains": { + "repoRuleId": "@@rules_oci+//oci/private:toolchains_repo.bzl%toolchains_repo", + "attributes": { + "toolchain_type": "@rules_oci//oci:regctl_toolchain_type", + "toolchain": "@oci_regctl_{platform}//:regctl_toolchain" + } + } + }, + "moduleExtensionMetadata": { + "explicitRootModuleDirectDeps": [], + "explicitRootModuleDirectDevDeps": [], + "useAllRepos": "NO", + "reproducible": false + }, + "recordedRepoMappingEntries": [ + [ + "aspect_bazel_lib+", + "bazel_tools", + "bazel_tools" + ], + [ + "bazel_features+", + "bazel_tools", + "bazel_tools" + ], + [ + "rules_oci+", + "aspect_bazel_lib", + "aspect_bazel_lib+" + ], + [ + "rules_oci+", + "bazel_features", + "bazel_features+" + ], + [ + "rules_oci+", + "bazel_skylib", + "bazel_skylib+" + ] + ] + } + }, "@@rules_python+//python/extensions:pip.bzl%pip": { "general": { "bzlTransitiveDigest": "w9qbx9gy25U04n5Tfmetw5ONAylCOEsMTo5DeljQ2GQ=", - "usagesDigest": "+Ag9bse4gNtMvORVLlx8Io3JqzUs5vS/HF3v1yP7+3Y=", + "usagesDigest": "Yxht2RNdONszrhtkJzz+mqdWbFsvWzyMXgp39D/NZ5M=", "recordedFileInputs": { "@@//requirements_lock.txt": "4aa61e82d368c885cfeba9eee9cd9b47b48994583ae913d0735b6c749743be10", "@@protobuf+//python/requirements.txt": "983be60d3cec4b319dcab6d48aeb3f5b2f7c3350f26b3a9e97486c37967c73c5", diff --git a/rules.bzl b/rules.bzl index 1517d5c..7886613 100644 --- a/rules.bzl +++ b/rules.bzl @@ -15,8 +15,8 @@ fi # --- begin runfiles.bash initialization v3 --- # Copy-pasted from the Bazel Bash runfiles library v3. set -uo pipefail; set +e; f=bazel_tools/tools/bash/runfiles/runfiles.bash -source $f || \ - source "${RUNFILES_DIR:-/dev/null}/$f" 2>/dev/null || \ +source "${RUNFILES_DIR:-/dev/null}/$f" 2>/dev/null || \ + source $f || \ source "$(grep -sm1 "^$f " "${RUNFILES_MANIFEST_FILE:-/dev/null}" | cut -f2- -d' ')" 2>/dev/null || \ source "$0.runfiles/$f" 2>/dev/null || \ source "$(grep -sm1 "^$f " "$0.runfiles_manifest" | cut -f2- -d' ')" 2>/dev/null || \ diff --git a/scripts/bb_remote b/scripts/bb_remote new file mode 100755 index 0000000..2914b19 --- /dev/null +++ b/scripts/bb_remote @@ -0,0 +1,5 @@ +#!/usr/bin/env bash + +eval $(bb_tmp) +echo "Building in $BUILD_HOST:$REMOTE_BUILD_PATH" +ssh $BUILD_HOST "cd $REMOTE_BUILD_PATH; . ~/.bash_profile; bazel $@" diff --git a/scripts/bb_test_all b/scripts/bb_test_all new file mode 100755 index 0000000..8792c86 --- /dev/null +++ b/scripts/bb_test_all @@ -0,0 +1,11 @@ +#!/usr/bin/env bash + +ROOT_DIR=$(pwd) +eval $(bb_tmp) +bazel test //... +ANY_FAILED=0 +for EXAMPLE in $(ls examples); do + echo "\nTesting examples/$EXAMPLE" + cd $ROOT_DIR/examples/$EXAMPLE + bazel test //... +done diff --git a/scripts/bb_tmp b/scripts/bb_tmp new file mode 100755 index 0000000..0d840de --- /dev/null +++ b/scripts/bb_tmp @@ -0,0 +1,10 @@ +#!/usr/bin/env bash + +BUILD_HOST=lab.axb + +# Create a tmp dir on the build machine +REMOTE_BUILD_PATH=$(ssh $BUILD_HOST -f 'mkdir -p /tmp/databuild && echo /tmp/databuild # mktemp --directory --tmpdir databuild/XXXX') +rsync -azh --filter=':- .gitignore' --exclude .git ./ $BUILD_HOST:$REMOTE_BUILD_PATH + +echo "BUILD_HOST=$BUILD_HOST" +echo "REMOTE_BUILD_PATH=$REMOTE_BUILD_PATH"