stuart/databuild
1
1
Fork 0
Code Issues 1 Pull requests Projects Releases Packages Wiki Activity Actions

commit
Some checks are pending
/ setup (push) Waiting to run
Details

Browse source
This commit is contained in:
Stuart Axelbrooke 2025-06-29 19:28:02 -07:00
parent 0c38569e71
commit ecee54da9f
11 changed files with 2915 additions and 1347 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
.bazelversion
View file

@ -1 +1 @@
8.2.1
8.3.0

3
.envrc
View file

@ -1,2 +1,5 @@
PATH_add scripts/
. .venv/bin/activate

54
MODULE.bazel
View file

@ -6,20 +6,54 @@ module(
bazel_dep(name = "bazel_skylib", version = "1.7.1")
bazel_dep(name = "platforms", version = "0.0.11")
bazel_dep(name = "rules_shell", version = "0.4.0")
bazel_dep(name = "rules_go", version = "0.46.0")
bazel_dep(name = "rules_oci", version = "2.2.6")
bazel_dep(name = "aspect_bazel_lib", version = "2.14.0")
bazel_dep(name = "rules_rust", version = "0.61.0")
crate = use_extension("@rules_rust//crate_universe:extensions.bzl", "crate")
crate.spec(package = "serde", features = ["derive"], version = "1.0")
crate.spec(package = "serde_json", version = "1.0")
crate.spec(package = "log", version = "0.4")
crate.spec(package = "simple_logger", features=["stderr"], version = "4.3")
crate.spec(package = "crossbeam-channel", version = "0.5")
crate.spec(package = "num_cpus", version = "1.16")
crate.spec(package = "tokio", default_features = False, features = ["macros", "net", "rt-multi-thread"], version = "1.38")
crate.spec(package = "prost", version = "0.13")
crate.spec(package = "prost-types", version = "0.13")
crate.spec(
features = ["derive"],
package = "serde",
version = "1.0",
)
crate.spec(
package = "serde_json",
version = "1.0",
)
crate.spec(
package = "log",
version = "0.4",
)
crate.spec(
features = ["stderr"],
package = "simple_logger",
version = "4.3",
)
crate.spec(
package = "crossbeam-channel",
version = "0.5",
)
crate.spec(
package = "num_cpus",
version = "1.16",
)
crate.spec(
default_features = False,
features = [
"macros",
"net",
"rt-multi-thread",
],
package = "tokio",
version = "1.38",
)
crate.spec(
package = "prost",
version = "0.13",
)
crate.spec(
package = "prost-types",
version = "0.13",
)
crate.from_specs()
use_repo(crate, "crates")

3381
MODULE.bazel.lock
View file

File diff suppressed because one or more lines are too long

1
databuild/graph/BUILD.bazel
View file

@ -1,4 +1,3 @@
load("@rules_go//go:def.bzl", "go_binary")
load("@rules_rust//rust:defs.bzl", "rust_binary", "rust_library")
exports_files([

7
examples/basic_graph/MODULE.bazel
View file

@ -42,7 +42,10 @@ oci = use_extension("@rules_oci//oci:extensions.bzl", "oci")
oci.pull(
name = "debian",
image = "docker.io/library/python",
platforms = ["linux/arm64/v8", "linux/amd64"],
platforms = [
"linux/arm64/v8",
"linux/amd64",
],
# 'latest' is not reproducible, but it's convenient.
# During the build we print a WARNING message that includes recommended 'digest' and 'platforms'
# values which you can use here in place of 'tag' to pin for reproducibility.
@ -50,7 +53,7 @@ oci.pull(
)
# For each oci.pull call, repeat the "name" here to expose them as dependencies.
use_repo(oci, "debian", "debian_linux_arm64_v8")
use_repo(oci, "debian", "debian_linux_amd64", "debian_linux_arm64_v8")
# Platforms for specifying linux/arm
bazel_dep(name = "platforms", version = "0.0.11")

368
examples/basic_graph/MODULE.bazel.lock
View file

File diff suppressed because one or more lines are too long

211
examples/basic_job/MODULE.bazel.lock
View file

@ -15,7 +15,6 @@
"https://bcr.bazel.build/modules/aspect_bazel_lib/2.14.0/MODULE.bazel": "2b31ffcc9bdc8295b2167e07a757dbbc9ac8906e7028e5170a3708cecaac119f",
"https://bcr.bazel.build/modules/aspect_bazel_lib/2.14.0/source.json": "0cf1826853b0bef8b5cd19c0610d717500f5521aa2b38b72b2ec302ac5e7526c",
"https://bcr.bazel.build/modules/aspect_bazel_lib/2.7.2/MODULE.bazel": "780d1a6522b28f5edb7ea09630748720721dfe27690d65a2d33aa7509de77e07",
"https://bcr.bazel.build/modules/bazel_features/1.1.0/MODULE.bazel": "cfd42ff3b815a5f39554d97182657f8c4b9719568eb7fded2b9135f084bf760b",
"https://bcr.bazel.build/modules/bazel_features/1.1.1/MODULE.bazel": "27b8c79ef57efe08efccbd9dd6ef70d61b4798320b8d3c134fd571f78963dbcd",
"https://bcr.bazel.build/modules/bazel_features/1.10.0/MODULE.bazel": "f75e8807570484a99be90abcd52b5e1f390362c258bcb73106f4544957a48101",
"https://bcr.bazel.build/modules/bazel_features/1.11.0/MODULE.bazel": "f9382337dd5a474c3b7d334c2f83e50b6eaedc284253334cf823044a26de03e8",
@ -24,7 +23,8 @@
"https://bcr.bazel.build/modules/bazel_features/1.18.0/MODULE.bazel": "1be0ae2557ab3a72a57aeb31b29be347bcdc5d2b1eb1e70f39e3851a7e97041a",
"https://bcr.bazel.build/modules/bazel_features/1.19.0/MODULE.bazel": "59adcdf28230d220f0067b1f435b8537dd033bfff8db21335ef9217919c7fb58",
"https://bcr.bazel.build/modules/bazel_features/1.21.0/MODULE.bazel": "675642261665d8eea09989aa3b8afb5c37627f1be178382c320d1b46afba5e3b",
"https://bcr.bazel.build/modules/bazel_features/1.21.0/source.json": "3e8379efaaef53ce35b7b8ba419df829315a880cb0a030e5bb45c96d6d5ecb5f",
"https://bcr.bazel.build/modules/bazel_features/1.30.0/MODULE.bazel": "a14b62d05969a293b80257e72e597c2da7f717e1e69fa8b339703ed6731bec87",
"https://bcr.bazel.build/modules/bazel_features/1.30.0/source.json": "b07e17f067fe4f69f90b03b36ef1e08fe0d1f3cac254c1241a1818773e3423bc",
"https://bcr.bazel.build/modules/bazel_features/1.4.1/MODULE.bazel": "e45b6bb2350aff3e442ae1111c555e27eac1d915e77775f6fdc4b351b758b5d7",
"https://bcr.bazel.build/modules/bazel_features/1.9.0/MODULE.bazel": "885151d58d90d8d9c811eb75e3288c11f850e1d6b481a8c9f766adee4712358b",
"https://bcr.bazel.build/modules/bazel_features/1.9.1/MODULE.bazel": "8f679097876a9b609ad1f60249c49d68bfab783dd9be012faf9d82547b14815a",
@ -42,10 +42,6 @@
"https://bcr.bazel.build/modules/bazel_skylib/1.7.1/source.json": "f121b43eeefc7c29efbd51b83d08631e2347297c95aac9764a701f2a6a2bb953",
"https://bcr.bazel.build/modules/buildozer/7.1.2/MODULE.bazel": "2e8dd40ede9c454042645fd8d8d0cd1527966aa5c919de86661e62953cd73d84",
"https://bcr.bazel.build/modules/buildozer/7.1.2/source.json": "c9028a501d2db85793a6996205c8de120944f50a0d570438fcae0457a5f9d1f8",
"https://bcr.bazel.build/modules/gazelle/0.32.0/MODULE.bazel": "b499f58a5d0d3537f3cf5b76d8ada18242f64ec474d8391247438bf04f58c7b8",
"https://bcr.bazel.build/modules/gazelle/0.33.0/MODULE.bazel": "a13a0f279b462b784fb8dd52a4074526c4a2afe70e114c7d09066097a46b3350",
"https://bcr.bazel.build/modules/gazelle/0.34.0/MODULE.bazel": "abdd8ce4d70978933209db92e436deb3a8b737859e9354fb5fd11fb5c2004c8a",
"https://bcr.bazel.build/modules/gazelle/0.34.0/source.json": "cdf0182297e3adabbdea2da88d5b930b2ee5e56511c3e7d6512069db6315a1f7",
"https://bcr.bazel.build/modules/google_benchmark/1.8.2/MODULE.bazel": "a70cf1bba851000ba93b58ae2f6d76490a9feb74192e57ab8e8ff13c34ec50cb",
"https://bcr.bazel.build/modules/googletest/1.11.0/MODULE.bazel": "3a83f095183f66345ca86aa13c58b59f9f94a2f81999c093d4eeaa2d262d12f4",
"https://bcr.bazel.build/modules/googletest/1.14.0.bcr.1/MODULE.bazel": "22c31a561553727960057361aa33bf20fb2e98584bc4fec007906e27053f80c6",
@ -70,8 +66,6 @@
"https://bcr.bazel.build/modules/protobuf/29.0/MODULE.bazel": "319dc8bf4c679ff87e71b1ccfb5a6e90a6dbc4693501d471f48662ac46d04e4e",
"https://bcr.bazel.build/modules/protobuf/29.0/source.json": "b857f93c796750eef95f0d61ee378f3420d00ee1dd38627b27193aa482f4f981",
"https://bcr.bazel.build/modules/protobuf/3.19.0/MODULE.bazel": "6b5fbb433f760a99a22b18b6850ed5784ef0e9928a72668b66e4d7ccd47db9b0",
"https://bcr.bazel.build/modules/protobuf/3.19.2/MODULE.bazel": "532ffe5f2186b69fdde039efe6df13ba726ff338c6bc82275ad433013fa10573",
"https://bcr.bazel.build/modules/protobuf/3.19.6/MODULE.bazel": "9233edc5e1f2ee276a60de3eaa47ac4132302ef9643238f23128fea53ea12858",
"https://bcr.bazel.build/modules/pybind11_bazel/2.11.1/MODULE.bazel": "88af1c246226d87e65be78ed49ecd1e6f5e98648558c14ce99176da041dc378e",
"https://bcr.bazel.build/modules/pybind11_bazel/2.11.1/source.json": "be4789e951dd5301282729fe3d4938995dc4c1a81c2ff150afc9f1b0504c6022",
"https://bcr.bazel.build/modules/re2/2023-09-01/MODULE.bazel": "cb3d511531b16cfc78a225a9e2136007a48cf8a677e4264baeab57fe78a80206",
@ -84,7 +78,6 @@
"https://bcr.bazel.build/modules/rules_cc/0.0.14/MODULE.bazel": "5e343a3aac88b8d7af3b1b6d2093b55c347b8eefc2e7d1442f7a02dc8fea48ac",
"https://bcr.bazel.build/modules/rules_cc/0.0.15/MODULE.bazel": "6704c35f7b4a72502ee81f61bf88706b54f06b3cbe5558ac17e2e14666cd5dcc",
"https://bcr.bazel.build/modules/rules_cc/0.0.16/MODULE.bazel": "7661303b8fc1b4d7f532e54e9d6565771fea666fbdf839e0a86affcd02defe87",
"https://bcr.bazel.build/modules/rules_cc/0.0.17/MODULE.bazel": "2ae1d8f4238ec67d7185d8861cb0a2cdf4bc608697c331b95bf990e69b62e64a",
"https://bcr.bazel.build/modules/rules_cc/0.0.2/MODULE.bazel": "6915987c90970493ab97393024c156ea8fb9f3bea953b2f3ec05c34f19b5695c",
"https://bcr.bazel.build/modules/rules_cc/0.0.6/MODULE.bazel": "abf360251023dfe3efcef65ab9d56beefa8394d4176dd29529750e1c57eaa33f",
"https://bcr.bazel.build/modules/rules_cc/0.0.8/MODULE.bazel": "964c85c82cfeb6f3855e6a07054fdb159aced38e99a5eecf7bce9d53990afa3e",
@ -94,10 +87,6 @@
"https://bcr.bazel.build/modules/rules_foreign_cc/0.9.0/MODULE.bazel": "c9e8c682bf75b0e7c704166d79b599f93b72cfca5ad7477df596947891feeef6",
"https://bcr.bazel.build/modules/rules_fuzzing/0.5.2/MODULE.bazel": "40c97d1144356f52905566c55811f13b299453a14ac7769dfba2ac38192337a8",
"https://bcr.bazel.build/modules/rules_fuzzing/0.5.2/source.json": "c8b1e2c717646f1702290959a3302a178fb639d987ab61d548105019f11e527e",
"https://bcr.bazel.build/modules/rules_go/0.41.0/MODULE.bazel": "55861d8e8bb0e62cbd2896f60ff303f62ffcb0eddb74ecb0e5c0cbe36fc292c8",
"https://bcr.bazel.build/modules/rules_go/0.42.0/MODULE.bazel": "8cfa875b9aa8c6fce2b2e5925e73c1388173ea3c32a0db4d2b4804b453c14270",
"https://bcr.bazel.build/modules/rules_go/0.46.0/MODULE.bazel": "3477df8bdcc49e698b9d25f734c4f3a9f5931ff34ee48a2c662be168f5f2d3fd",
"https://bcr.bazel.build/modules/rules_go/0.46.0/source.json": "fbf0e50e8ed487272e5c0977c0b67c74cbe97e1880b45bbeff44a3338dc8a08e",
"https://bcr.bazel.build/modules/rules_java/4.0.0/MODULE.bazel": "5a78a7ae82cd1a33cef56dc578c7d2a46ed0dca12643ee45edbb8417899e6f74",
"https://bcr.bazel.build/modules/rules_java/5.3.5/MODULE.bazel": "a4ec4f2db570171e3e5eb753276ee4b389bae16b96207e9d3230895c99644b86",
"https://bcr.bazel.build/modules/rules_java/6.0.0/MODULE.bazel": "8a43b7df601a7ec1af61d79345c17b31ea1fedc6711fd4abfd013ea612978e39",
@ -109,8 +98,8 @@
"https://bcr.bazel.build/modules/rules_java/7.2.0/MODULE.bazel": "06c0334c9be61e6cef2c8c84a7800cef502063269a5af25ceb100b192453d4ab",
"https://bcr.bazel.build/modules/rules_java/7.3.2/MODULE.bazel": "50dece891cfdf1741ea230d001aa9c14398062f2b7c066470accace78e412bc2",
"https://bcr.bazel.build/modules/rules_java/7.6.1/MODULE.bazel": "2f14b7e8a1aa2f67ae92bc69d1ec0fa8d9f827c4e17ff5e5f02e91caa3b2d0fe",
"https://bcr.bazel.build/modules/rules_java/8.11.0/MODULE.bazel": "c3d280bc5ff1038dcb3bacb95d3f6b83da8dd27bba57820ec89ea4085da767ad",
"https://bcr.bazel.build/modules/rules_java/8.11.0/source.json": "302b52a39259a85aa06ca3addb9787864ca3e03b432a5f964ea68244397e7544",
"https://bcr.bazel.build/modules/rules_java/8.12.0/MODULE.bazel": "8e6590b961f2defdfc2811c089c75716cb2f06c8a4edeb9a8d85eaa64ee2a761",
"https://bcr.bazel.build/modules/rules_java/8.12.0/source.json": "cbd5d55d9d38d4008a7d00bee5b5a5a4b6031fcd4a56515c9accbcd42c7be2ba",
"https://bcr.bazel.build/modules/rules_jvm_external/4.4.2/MODULE.bazel": "a56b85e418c83eb1839819f0b515c431010160383306d13ec21959ac412d2fe7",
"https://bcr.bazel.build/modules/rules_jvm_external/5.1/MODULE.bazel": "33f6f999e03183f7d088c9be518a63467dfd0be94a11d0055fe2d210f89aa909",
"https://bcr.bazel.build/modules/rules_jvm_external/5.2/MODULE.bazel": "d9351ba35217ad0de03816ef3ed63f89d411349353077348a45348b096615036",
@ -159,9 +148,8 @@
"https://bcr.bazel.build/modules/stardoc/0.7.1/source.json": "b6500ffcd7b48cd72c29bb67bcac781e12701cc0d6d55d266a652583cfcdab01",
"https://bcr.bazel.build/modules/upb/0.0.0-20220923-a547704/MODULE.bazel": "7298990c00040a0e2f121f6c32544bab27d4452f80d9ce51349b1a28f3005c43",
"https://bcr.bazel.build/modules/zlib/1.2.11/MODULE.bazel": "07b389abc85fdbca459b69e2ec656ae5622873af3f845e1c9d80fe179f3effa0",
"https://bcr.bazel.build/modules/zlib/1.2.12/MODULE.bazel": "3b1a8834ada2a883674be8cbd36ede1b6ec481477ada359cd2d3ddc562340b27",
"https://bcr.bazel.build/modules/zlib/1.3.1.bcr.3/MODULE.bazel": "af322bc08976524477c79d1e45e241b6efbeb918c497e8840b8ab116802dda79",
"https://bcr.bazel.build/modules/zlib/1.3.1.bcr.3/source.json": "2be409ac3c7601245958cd4fcdff4288be79ed23bd690b4b951f500d54ee6e7d",
"https://bcr.bazel.build/modules/zlib/1.3.1.bcr.5/MODULE.bazel": "eec517b5bbe5492629466e11dae908d043364302283de25581e3eb944326c4ca",
"https://bcr.bazel.build/modules/zlib/1.3.1.bcr.5/source.json": "22bc55c47af97246cfc093d0acf683a7869377de362b5d1c552c2c2e16b7a806",
"https://bcr.bazel.build/modules/zlib/1.3.1/MODULE.bazel": "751c9940dcfe869f5f7274e1295422a34623555916eb98c174c1e945594bf198"
},
"selectedYankedVersions": {},
@ -197,192 +185,9 @@
]
}
},
"@@rules_go+//go:extensions.bzl%go_sdk": {
"os:osx,arch:aarch64": {
"bzlTransitiveDigest": "jBP0cRKOr+A42aPGunoasOD+vrmMLJIJ8Jwi65DdelE=",
"usagesDigest": "9WUdtwMNxQMIp54gOxEBVws3WnIUdQcvX9pRfBtrtvQ=",
"recordedFileInputs": {},
"recordedDirentsInputs": {},
"envVariables": {},
"generatedRepoSpecs": {
"io_bazel_rules_nogo": {
"repoRuleId": "@@rules_go+//go/private:nogo.bzl%go_register_nogo",
"attributes": {
"nogo": "@io_bazel_rules_go//:default_nogo",
"includes": [
"'@@//:__subpackages__'"
],
"excludes": []
}
},
"go_default_sdk": {
"repoRuleId": "@@rules_go+//go/private:sdk.bzl%go_download_sdk_rule",
"attributes": {
"goos": "",
"goarch": "",
"sdks": {},
"experiments": [],
"patches": [],
"patch_strip": 0,
"urls": [
"https://dl.google.com/go/{}"
],
"version": "1.21.1",
"strip_prefix": "go"
}
},
"rules_go__download_0_darwin_amd64": {
"repoRuleId": "@@rules_go+//go/private:sdk.bzl%go_download_sdk_rule",
"attributes": {
"goos": "",
"goarch": "",
"sdks": {},
"urls": [
"https://dl.google.com/go/{}"
],
"version": "1.21.1"
}
},
"rules_go__download_0_linux_amd64": {
"repoRuleId": "@@rules_go+//go/private:sdk.bzl%go_download_sdk_rule",
"attributes": {
"goos": "",
"goarch": "",
"sdks": {},
"urls": [
"https://dl.google.com/go/{}"
],
"version": "1.21.1"
}
},
"rules_go__download_0_linux_arm64": {
"repoRuleId": "@@rules_go+//go/private:sdk.bzl%go_download_sdk_rule",
"attributes": {
"goos": "",
"goarch": "",
"sdks": {},
"urls": [
"https://dl.google.com/go/{}"
],
"version": "1.21.1"
}
},
"rules_go__download_0_windows_amd64": {
"repoRuleId": "@@rules_go+//go/private:sdk.bzl%go_download_sdk_rule",
"attributes": {
"goos": "",
"goarch": "",
"sdks": {},
"urls": [
"https://dl.google.com/go/{}"
],
"version": "1.21.1"
}
},
"rules_go__download_0_windows_arm64": {
"repoRuleId": "@@rules_go+//go/private:sdk.bzl%go_download_sdk_rule",
"attributes": {
"goos": "",
"goarch": "",
"sdks": {},
"urls": [
"https://dl.google.com/go/{}"
],
"version": "1.21.1"
}
},
"go_host_compatible_sdk_label": {
"repoRuleId": "@@rules_go+//go/private:extensions.bzl%host_compatible_toolchain",
"attributes": {
"toolchain": "@go_default_sdk//:ROOT"
}
},
"go_toolchains": {
"repoRuleId": "@@rules_go+//go/private:sdk.bzl%go_multiple_toolchains",
"attributes": {
"prefixes": [
"_0000_go_default_sdk_",
"_0001_rules_go__download_0_darwin_amd64_",
"_0002_rules_go__download_0_linux_amd64_",
"_0003_rules_go__download_0_linux_arm64_",
"_0004_rules_go__download_0_windows_amd64_",
"_0005_rules_go__download_0_windows_arm64_"
],
"geese": [
"",
"darwin",
"linux",
"linux",
"windows",
"windows"
],
"goarchs": [
"",
"amd64",
"amd64",
"arm64",
"amd64",
"arm64"
],
"sdk_repos": [
"go_default_sdk",
"rules_go__download_0_darwin_amd64",
"rules_go__download_0_linux_amd64",
"rules_go__download_0_linux_arm64",
"rules_go__download_0_windows_amd64",
"rules_go__download_0_windows_arm64"
],
"sdk_types": [
"remote",
"remote",
"remote",
"remote",
"remote",
"remote"
],
"sdk_versions": [
"1.21.1",
"1.21.1",
"1.21.1",
"1.21.1",
"1.21.1",
"1.21.1"
]
}
}
},
"recordedRepoMappingEntries": [
[
"bazel_features+",
"bazel_features_globals",
"bazel_features++version_extension+bazel_features_globals"
],
[
"bazel_features+",
"bazel_features_version",
"bazel_features++version_extension+bazel_features_version"
],
[
"rules_go+",
"bazel_tools",
"bazel_tools"
],
[
"rules_go+",
"io_bazel_rules_go",
"rules_go+"
],
[
"rules_go+",
"io_bazel_rules_go_bazel_features",
"bazel_features+"
]
]
}
},
"@@rules_kotlin+//src/main/starlark/core/repositories:bzlmod_setup.bzl%rules_kotlin_extensions": {
"general": {
"bzlTransitiveDigest": "sFhcgPbDQehmbD1EOXzX4H1q/CD5df8zwG4kp4jbvr8=",
"bzlTransitiveDigest": "hUTp2w+RUVdL7ma5esCXZJAFnX7vLbVfLd7FwnQI6bU=",
"usagesDigest": "QI2z8ZUR+mqtbwsf2fLqYdJAkPOHdOV+tF2yVAUgRzw=",
"recordedFileInputs": {},
"recordedDirentsInputs": {},
@ -446,7 +251,7 @@
},
"@@rules_oci+//oci:extensions.bzl%oci": {
"general": {
"bzlTransitiveDigest": "8fB61KHYOU4XHH65DqVw59ZIDGO29I2WIbVHxii4slA=",
"bzlTransitiveDigest": "KHcdN2ovRQGX1MKsH0nGoGPFd/84U43tssN2jImCeJU=",
"usagesDigest": "/O1PwnnkqSBmI9Oe08ZYYqjM4IS8JR+/9rjgzVTNDaQ=",
"recordedFileInputs": {},
"recordedDirentsInputs": {},

213
examples/podcast_reviews/MODULE.bazel.lock
View file

@ -15,7 +15,6 @@
"https://bcr.bazel.build/modules/aspect_bazel_lib/2.14.0/MODULE.bazel": "2b31ffcc9bdc8295b2167e07a757dbbc9ac8906e7028e5170a3708cecaac119f",
"https://bcr.bazel.build/modules/aspect_bazel_lib/2.14.0/source.json": "0cf1826853b0bef8b5cd19c0610d717500f5521aa2b38b72b2ec302ac5e7526c",
"https://bcr.bazel.build/modules/aspect_bazel_lib/2.7.2/MODULE.bazel": "780d1a6522b28f5edb7ea09630748720721dfe27690d65a2d33aa7509de77e07",
"https://bcr.bazel.build/modules/bazel_features/1.1.0/MODULE.bazel": "cfd42ff3b815a5f39554d97182657f8c4b9719568eb7fded2b9135f084bf760b",
"https://bcr.bazel.build/modules/bazel_features/1.1.1/MODULE.bazel": "27b8c79ef57efe08efccbd9dd6ef70d61b4798320b8d3c134fd571f78963dbcd",
"https://bcr.bazel.build/modules/bazel_features/1.10.0/MODULE.bazel": "f75e8807570484a99be90abcd52b5e1f390362c258bcb73106f4544957a48101",
"https://bcr.bazel.build/modules/bazel_features/1.11.0/MODULE.bazel": "f9382337dd5a474c3b7d334c2f83e50b6eaedc284253334cf823044a26de03e8",
@ -24,7 +23,8 @@
"https://bcr.bazel.build/modules/bazel_features/1.18.0/MODULE.bazel": "1be0ae2557ab3a72a57aeb31b29be347bcdc5d2b1eb1e70f39e3851a7e97041a",
"https://bcr.bazel.build/modules/bazel_features/1.19.0/MODULE.bazel": "59adcdf28230d220f0067b1f435b8537dd033bfff8db21335ef9217919c7fb58",
"https://bcr.bazel.build/modules/bazel_features/1.21.0/MODULE.bazel": "675642261665d8eea09989aa3b8afb5c37627f1be178382c320d1b46afba5e3b",
"https://bcr.bazel.build/modules/bazel_features/1.21.0/source.json": "3e8379efaaef53ce35b7b8ba419df829315a880cb0a030e5bb45c96d6d5ecb5f",
"https://bcr.bazel.build/modules/bazel_features/1.30.0/MODULE.bazel": "a14b62d05969a293b80257e72e597c2da7f717e1e69fa8b339703ed6731bec87",
"https://bcr.bazel.build/modules/bazel_features/1.30.0/source.json": "b07e17f067fe4f69f90b03b36ef1e08fe0d1f3cac254c1241a1818773e3423bc",
"https://bcr.bazel.build/modules/bazel_features/1.4.1/MODULE.bazel": "e45b6bb2350aff3e442ae1111c555e27eac1d915e77775f6fdc4b351b758b5d7",
"https://bcr.bazel.build/modules/bazel_features/1.9.0/MODULE.bazel": "885151d58d90d8d9c811eb75e3288c11f850e1d6b481a8c9f766adee4712358b",
"https://bcr.bazel.build/modules/bazel_features/1.9.1/MODULE.bazel": "8f679097876a9b609ad1f60249c49d68bfab783dd9be012faf9d82547b14815a",
@ -42,10 +42,6 @@
"https://bcr.bazel.build/modules/bazel_skylib/1.7.1/source.json": "f121b43eeefc7c29efbd51b83d08631e2347297c95aac9764a701f2a6a2bb953",
"https://bcr.bazel.build/modules/buildozer/7.1.2/MODULE.bazel": "2e8dd40ede9c454042645fd8d8d0cd1527966aa5c919de86661e62953cd73d84",
"https://bcr.bazel.build/modules/buildozer/7.1.2/source.json": "c9028a501d2db85793a6996205c8de120944f50a0d570438fcae0457a5f9d1f8",
"https://bcr.bazel.build/modules/gazelle/0.32.0/MODULE.bazel": "b499f58a5d0d3537f3cf5b76d8ada18242f64ec474d8391247438bf04f58c7b8",
"https://bcr.bazel.build/modules/gazelle/0.33.0/MODULE.bazel": "a13a0f279b462b784fb8dd52a4074526c4a2afe70e114c7d09066097a46b3350",
"https://bcr.bazel.build/modules/gazelle/0.34.0/MODULE.bazel": "abdd8ce4d70978933209db92e436deb3a8b737859e9354fb5fd11fb5c2004c8a",
"https://bcr.bazel.build/modules/gazelle/0.34.0/source.json": "cdf0182297e3adabbdea2da88d5b930b2ee5e56511c3e7d6512069db6315a1f7",
"https://bcr.bazel.build/modules/google_benchmark/1.8.2/MODULE.bazel": "a70cf1bba851000ba93b58ae2f6d76490a9feb74192e57ab8e8ff13c34ec50cb",
"https://bcr.bazel.build/modules/googletest/1.11.0/MODULE.bazel": "3a83f095183f66345ca86aa13c58b59f9f94a2f81999c093d4eeaa2d262d12f4",
"https://bcr.bazel.build/modules/googletest/1.14.0.bcr.1/MODULE.bazel": "22c31a561553727960057361aa33bf20fb2e98584bc4fec007906e27053f80c6",
@ -71,8 +67,6 @@
"https://bcr.bazel.build/modules/protobuf/29.0/MODULE.bazel": "319dc8bf4c679ff87e71b1ccfb5a6e90a6dbc4693501d471f48662ac46d04e4e",
"https://bcr.bazel.build/modules/protobuf/29.0/source.json": "b857f93c796750eef95f0d61ee378f3420d00ee1dd38627b27193aa482f4f981",
"https://bcr.bazel.build/modules/protobuf/3.19.0/MODULE.bazel": "6b5fbb433f760a99a22b18b6850ed5784ef0e9928a72668b66e4d7ccd47db9b0",
"https://bcr.bazel.build/modules/protobuf/3.19.2/MODULE.bazel": "532ffe5f2186b69fdde039efe6df13ba726ff338c6bc82275ad433013fa10573",
"https://bcr.bazel.build/modules/protobuf/3.19.6/MODULE.bazel": "9233edc5e1f2ee276a60de3eaa47ac4132302ef9643238f23128fea53ea12858",
"https://bcr.bazel.build/modules/pybind11_bazel/2.11.1/MODULE.bazel": "88af1c246226d87e65be78ed49ecd1e6f5e98648558c14ce99176da041dc378e",
"https://bcr.bazel.build/modules/pybind11_bazel/2.11.1/source.json": "be4789e951dd5301282729fe3d4938995dc4c1a81c2ff150afc9f1b0504c6022",
"https://bcr.bazel.build/modules/re2/2023-09-01/MODULE.bazel": "cb3d511531b16cfc78a225a9e2136007a48cf8a677e4264baeab57fe78a80206",
@ -85,7 +79,6 @@
"https://bcr.bazel.build/modules/rules_cc/0.0.14/MODULE.bazel": "5e343a3aac88b8d7af3b1b6d2093b55c347b8eefc2e7d1442f7a02dc8fea48ac",
"https://bcr.bazel.build/modules/rules_cc/0.0.15/MODULE.bazel": "6704c35f7b4a72502ee81f61bf88706b54f06b3cbe5558ac17e2e14666cd5dcc",
"https://bcr.bazel.build/modules/rules_cc/0.0.16/MODULE.bazel": "7661303b8fc1b4d7f532e54e9d6565771fea666fbdf839e0a86affcd02defe87",
"https://bcr.bazel.build/modules/rules_cc/0.0.17/MODULE.bazel": "2ae1d8f4238ec67d7185d8861cb0a2cdf4bc608697c331b95bf990e69b62e64a",
"https://bcr.bazel.build/modules/rules_cc/0.0.2/MODULE.bazel": "6915987c90970493ab97393024c156ea8fb9f3bea953b2f3ec05c34f19b5695c",
"https://bcr.bazel.build/modules/rules_cc/0.0.6/MODULE.bazel": "abf360251023dfe3efcef65ab9d56beefa8394d4176dd29529750e1c57eaa33f",
"https://bcr.bazel.build/modules/rules_cc/0.0.8/MODULE.bazel": "964c85c82cfeb6f3855e6a07054fdb159aced38e99a5eecf7bce9d53990afa3e",
@ -95,10 +88,6 @@
"https://bcr.bazel.build/modules/rules_foreign_cc/0.9.0/MODULE.bazel": "c9e8c682bf75b0e7c704166d79b599f93b72cfca5ad7477df596947891feeef6",
"https://bcr.bazel.build/modules/rules_fuzzing/0.5.2/MODULE.bazel": "40c97d1144356f52905566c55811f13b299453a14ac7769dfba2ac38192337a8",
"https://bcr.bazel.build/modules/rules_fuzzing/0.5.2/source.json": "c8b1e2c717646f1702290959a3302a178fb639d987ab61d548105019f11e527e",
"https://bcr.bazel.build/modules/rules_go/0.41.0/MODULE.bazel": "55861d8e8bb0e62cbd2896f60ff303f62ffcb0eddb74ecb0e5c0cbe36fc292c8",
"https://bcr.bazel.build/modules/rules_go/0.42.0/MODULE.bazel": "8cfa875b9aa8c6fce2b2e5925e73c1388173ea3c32a0db4d2b4804b453c14270",
"https://bcr.bazel.build/modules/rules_go/0.46.0/MODULE.bazel": "3477df8bdcc49e698b9d25f734c4f3a9f5931ff34ee48a2c662be168f5f2d3fd",
"https://bcr.bazel.build/modules/rules_go/0.46.0/source.json": "fbf0e50e8ed487272e5c0977c0b67c74cbe97e1880b45bbeff44a3338dc8a08e",
"https://bcr.bazel.build/modules/rules_java/4.0.0/MODULE.bazel": "5a78a7ae82cd1a33cef56dc578c7d2a46ed0dca12643ee45edbb8417899e6f74",
"https://bcr.bazel.build/modules/rules_java/5.3.5/MODULE.bazel": "a4ec4f2db570171e3e5eb753276ee4b389bae16b96207e9d3230895c99644b86",
"https://bcr.bazel.build/modules/rules_java/6.0.0/MODULE.bazel": "8a43b7df601a7ec1af61d79345c17b31ea1fedc6711fd4abfd013ea612978e39",
@ -110,8 +99,8 @@
"https://bcr.bazel.build/modules/rules_java/7.2.0/MODULE.bazel": "06c0334c9be61e6cef2c8c84a7800cef502063269a5af25ceb100b192453d4ab",
"https://bcr.bazel.build/modules/rules_java/7.3.2/MODULE.bazel": "50dece891cfdf1741ea230d001aa9c14398062f2b7c066470accace78e412bc2",
"https://bcr.bazel.build/modules/rules_java/7.6.1/MODULE.bazel": "2f14b7e8a1aa2f67ae92bc69d1ec0fa8d9f827c4e17ff5e5f02e91caa3b2d0fe",
"https://bcr.bazel.build/modules/rules_java/8.11.0/MODULE.bazel": "c3d280bc5ff1038dcb3bacb95d3f6b83da8dd27bba57820ec89ea4085da767ad",
"https://bcr.bazel.build/modules/rules_java/8.11.0/source.json": "302b52a39259a85aa06ca3addb9787864ca3e03b432a5f964ea68244397e7544",
"https://bcr.bazel.build/modules/rules_java/8.12.0/MODULE.bazel": "8e6590b961f2defdfc2811c089c75716cb2f06c8a4edeb9a8d85eaa64ee2a761",
"https://bcr.bazel.build/modules/rules_java/8.12.0/source.json": "cbd5d55d9d38d4008a7d00bee5b5a5a4b6031fcd4a56515c9accbcd42c7be2ba",
"https://bcr.bazel.build/modules/rules_java/8.3.2/MODULE.bazel": "7336d5511ad5af0b8615fdc7477535a2e4e723a357b6713af439fe8cf0195017",
"https://bcr.bazel.build/modules/rules_java/8.5.1/MODULE.bazel": "d8a9e38cc5228881f7055a6079f6f7821a073df3744d441978e7a43e20226939",
"https://bcr.bazel.build/modules/rules_jvm_external/4.4.2/MODULE.bazel": "a56b85e418c83eb1839819f0b515c431010160383306d13ec21959ac412d2fe7",
@ -164,9 +153,8 @@
"https://bcr.bazel.build/modules/stardoc/0.7.2/source.json": "58b029e5e901d6802967754adf0a9056747e8176f017cfe3607c0851f4d42216",
"https://bcr.bazel.build/modules/upb/0.0.0-20220923-a547704/MODULE.bazel": "7298990c00040a0e2f121f6c32544bab27d4452f80d9ce51349b1a28f3005c43",
"https://bcr.bazel.build/modules/zlib/1.2.11/MODULE.bazel": "07b389abc85fdbca459b69e2ec656ae5622873af3f845e1c9d80fe179f3effa0",
"https://bcr.bazel.build/modules/zlib/1.2.12/MODULE.bazel": "3b1a8834ada2a883674be8cbd36ede1b6ec481477ada359cd2d3ddc562340b27",
"https://bcr.bazel.build/modules/zlib/1.3.1.bcr.3/MODULE.bazel": "af322bc08976524477c79d1e45e241b6efbeb918c497e8840b8ab116802dda79",
"https://bcr.bazel.build/modules/zlib/1.3.1.bcr.3/source.json": "2be409ac3c7601245958cd4fcdff4288be79ed23bd690b4b951f500d54ee6e7d",
"https://bcr.bazel.build/modules/zlib/1.3.1.bcr.5/MODULE.bazel": "eec517b5bbe5492629466e11dae908d043364302283de25581e3eb944326c4ca",
"https://bcr.bazel.build/modules/zlib/1.3.1.bcr.5/source.json": "22bc55c47af97246cfc093d0acf683a7869377de362b5d1c552c2c2e16b7a806",
"https://bcr.bazel.build/modules/zlib/1.3.1/MODULE.bazel": "751c9940dcfe869f5f7274e1295422a34623555916eb98c174c1e945594bf198"
},
"selectedYankedVersions": {},
@ -202,192 +190,9 @@
]
}
},
"@@rules_go+//go:extensions.bzl%go_sdk": {
"os:osx,arch:aarch64": {
"bzlTransitiveDigest": "jBP0cRKOr+A42aPGunoasOD+vrmMLJIJ8Jwi65DdelE=",
"usagesDigest": "9WUdtwMNxQMIp54gOxEBVws3WnIUdQcvX9pRfBtrtvQ=",
"recordedFileInputs": {},
"recordedDirentsInputs": {},
"envVariables": {},
"generatedRepoSpecs": {
"io_bazel_rules_nogo": {
"repoRuleId": "@@rules_go+//go/private:nogo.bzl%go_register_nogo",
"attributes": {
"nogo": "@io_bazel_rules_go//:default_nogo",
"includes": [
"'@@//:__subpackages__'"
],
"excludes": []
}
},
"go_default_sdk": {
"repoRuleId": "@@rules_go+//go/private:sdk.bzl%go_download_sdk_rule",
"attributes": {
"goos": "",
"goarch": "",
"sdks": {},
"experiments": [],
"patches": [],
"patch_strip": 0,
"urls": [
"https://dl.google.com/go/{}"
],
"version": "1.21.1",
"strip_prefix": "go"
}
},
"rules_go__download_0_darwin_amd64": {
"repoRuleId": "@@rules_go+//go/private:sdk.bzl%go_download_sdk_rule",
"attributes": {
"goos": "",
"goarch": "",
"sdks": {},
"urls": [
"https://dl.google.com/go/{}"
],
"version": "1.21.1"
}
},
"rules_go__download_0_linux_amd64": {
"repoRuleId": "@@rules_go+//go/private:sdk.bzl%go_download_sdk_rule",
"attributes": {
"goos": "",
"goarch": "",
"sdks": {},
"urls": [
"https://dl.google.com/go/{}"
],
"version": "1.21.1"
}
},
"rules_go__download_0_linux_arm64": {
"repoRuleId": "@@rules_go+//go/private:sdk.bzl%go_download_sdk_rule",
"attributes": {
"goos": "",
"goarch": "",
"sdks": {},
"urls": [
"https://dl.google.com/go/{}"
],
"version": "1.21.1"
}
},
"rules_go__download_0_windows_amd64": {
"repoRuleId": "@@rules_go+//go/private:sdk.bzl%go_download_sdk_rule",
"attributes": {
"goos": "",
"goarch": "",
"sdks": {},
"urls": [
"https://dl.google.com/go/{}"
],
"version": "1.21.1"
}
},
"rules_go__download_0_windows_arm64": {
"repoRuleId": "@@rules_go+//go/private:sdk.bzl%go_download_sdk_rule",
"attributes": {
"goos": "",
"goarch": "",
"sdks": {},
"urls": [
"https://dl.google.com/go/{}"
],
"version": "1.21.1"
}
},
"go_host_compatible_sdk_label": {
"repoRuleId": "@@rules_go+//go/private:extensions.bzl%host_compatible_toolchain",
"attributes": {
"toolchain": "@go_default_sdk//:ROOT"
}
},
"go_toolchains": {
"repoRuleId": "@@rules_go+//go/private:sdk.bzl%go_multiple_toolchains",
"attributes": {
"prefixes": [
"_0000_go_default_sdk_",
"_0001_rules_go__download_0_darwin_amd64_",
"_0002_rules_go__download_0_linux_amd64_",
"_0003_rules_go__download_0_linux_arm64_",
"_0004_rules_go__download_0_windows_amd64_",
"_0005_rules_go__download_0_windows_arm64_"
],
"geese": [
"",
"darwin",
"linux",
"linux",
"windows",
"windows"
],
"goarchs": [
"",
"amd64",
"amd64",
"arm64",
"amd64",
"arm64"
],
"sdk_repos": [
"go_default_sdk",
"rules_go__download_0_darwin_amd64",
"rules_go__download_0_linux_amd64",
"rules_go__download_0_linux_arm64",
"rules_go__download_0_windows_amd64",
"rules_go__download_0_windows_arm64"
],
"sdk_types": [
"remote",
"remote",
"remote",
"remote",
"remote",
"remote"
],
"sdk_versions": [
"1.21.1",
"1.21.1",
"1.21.1",
"1.21.1",
"1.21.1",
"1.21.1"
]
}
}
},
"recordedRepoMappingEntries": [
[
"bazel_features+",
"bazel_features_globals",
"bazel_features++version_extension+bazel_features_globals"
],
[
"bazel_features+",
"bazel_features_version",
"bazel_features++version_extension+bazel_features_version"
],
[
"rules_go+",
"bazel_tools",
"bazel_tools"
],
[
"rules_go+",
"io_bazel_rules_go",
"rules_go+"
],
[
"rules_go+",
"io_bazel_rules_go_bazel_features",
"bazel_features+"
]
]
}
},
"@@rules_kotlin+//src/main/starlark/core/repositories:bzlmod_setup.bzl%rules_kotlin_extensions": {
"general": {
"bzlTransitiveDigest": "sFhcgPbDQehmbD1EOXzX4H1q/CD5df8zwG4kp4jbvr8=",
"bzlTransitiveDigest": "hUTp2w+RUVdL7ma5esCXZJAFnX7vLbVfLd7FwnQI6bU=",
"usagesDigest": "QI2z8ZUR+mqtbwsf2fLqYdJAkPOHdOV+tF2yVAUgRzw=",
"recordedFileInputs": {},
"recordedDirentsInputs": {},
@ -451,7 +256,7 @@
},
"@@rules_oci+//oci:extensions.bzl%oci": {
"general": {
"bzlTransitiveDigest": "8fB61KHYOU4XHH65DqVw59ZIDGO29I2WIbVHxii4slA=",
"bzlTransitiveDigest": "KHcdN2ovRQGX1MKsH0nGoGPFd/84U43tssN2jImCeJU=",
"usagesDigest": "/O1PwnnkqSBmI9Oe08ZYYqjM4IS8JR+/9rjgzVTNDaQ=",
"recordedFileInputs": {},
"recordedDirentsInputs": {},
@ -608,7 +413,7 @@
},
"@@rules_python+//python/extensions:pip.bzl%pip": {
"general": {
"bzlTransitiveDigest": "w9qbx9gy25U04n5Tfmetw5ONAylCOEsMTo5DeljQ2GQ=",
"bzlTransitiveDigest": "kSgIwtQrmQ9xXVatFAGiyEeTmBDJtfyphXilwN8+UKo=",
"usagesDigest": "Yxht2RNdONszrhtkJzz+mqdWbFsvWzyMXgp39D/NZ5M=",
"recordedFileInputs": {
"@@//requirements_lock.txt": "4aa61e82d368c885cfeba9eee9cd9b47b48994583ae913d0735b6c749743be10",

17
examples/podcast_reviews/README.md
View file

@ -1,6 +1,23 @@
# Podcast Reviews Example
This is an example data application where we produce text insights from podcast review data. It is made up of N datasets:
- Raw reviews `(date, podcast, text, rating)`
- Podcasts `(podcast, title, category)`
- Categorized review text `(date, category, podcast, text)`
- Phrase models `(date, category, hash, ngram, score)`
- Podcast phrase stats `(date, category, podcast, ngram, count, rating)`
- Podcast daily summary `(date, category, podcast, phrase_stats, recent_reviews)`
```mermaid
flowchart LR
raw_reviews[(Raw Reviews)] & podcasts[(Podcasts)] --> categorize_text --> categorized_texts[(Categorized Texts)]
categorized_texts --> phrase[Phrase Modeling] --> phrase_models[(Phrase Models)]
phrase_models & raw_reviews --> phrase_stats --> podcast_phrase_stats[(Podcast Phrase Stats)]
podcast_phrase_stats & raw_reviews --> calc_summary --> podcast_daily_summary[(Podcast Daily Summary)]
```
## Input Data
Get it from [here](https://www.kaggle.com/datasets/thoughtvector/podcastreviews/versions/28?select=database.sqlite)! (and put it in `examples/podcast_reviews/data/ingest/database.sqlite`)

5
manifesto.md
View file

@ -4,7 +4,7 @@
### Motivation
The modern ML company today is a data company, whose value is derived from ingesting, processing, and refining data. The vast majority of our data providers provide it in discrete batches, and we deliver discrete batches to our customers, and the flow/process this data goes through from consumption to vending is often long and complex. We have the opportunity to define a simple but expressive fabric that simply connects our data inputs to our data outputs in a principled manner, separating concerns by scale, and minimizing our operational overhead. This fabric would remove human judgment required to produce any data asset, and take responsibility for achieving the end to end data production process.
The modern ML company today is a data company, whose value is derived from ingesting, processing, and refining data. The vast majority of our data providers provide it in discrete batches, most interfaces where data is vended internally or to customers is via batches, and the flow/process this data goes through from consumption to vending is often long and complex. We have the opportunity to define a simple and declarative fabric that connects our data inputs to our data outputs in a principled manner, separating concerns by scale, and minimizing our operational overhead. This fabric would remove human judgment required to produce any data asset, and take responsibility for achieving the end-to-end data production process.
This fabric also allows us to completely define the scope of involved code and data for a given concern, increasing engineering velocity and quality. It also allows separating concerns by scale: keeping job internal logic separate from job/dataset composition logic, minimizing the number of things that must be considered when changing or authoring new code and data. It is also important to be practical, not dogmatic, and this system should thrive in an environment with experimentation and other orchestration strategies, so long as the assumptions below hold true.
@ -148,5 +148,4 @@ Notes:
### Data Catalog
The data catalog is an essential component that enables cache-aware planning of data builds. It is a mapping of `(partition_ref, mtime) -> partition_manifest`. Access to the described data graph and the data catalog is all that is needed to plan out the net work needed to materialize a new partition. Jobs themselves are responsible for any short-circuiting for work that happens out of band.
## Entity Relationships
Note: Data services are missing below. They could be "libraries" for data, e.g. filling an organizational role for data dependencies at a larger scale.
##