diff --git a/examples/basic_graph/DataDep.java b/examples/basic_graph/DataDep.java index 5fb2161..6a67f44 100644 --- a/examples/basic_graph/DataDep.java +++ b/examples/basic_graph/DataDep.java @@ -1,7 +1,6 @@ package com.databuild.examples.basic_graph; public class DataDep { - private String depType; // "query" or "materialize" - private String ref; - // getters, setters, constructors + public String depType; // "query" or "materialize" + public String ref; } diff --git a/examples/basic_graph/GenerateConfigure.java b/examples/basic_graph/GenerateConfigure.java index 520bc04..f052dad 100644 --- a/examples/basic_graph/GenerateConfigure.java +++ b/examples/basic_graph/GenerateConfigure.java @@ -3,6 +3,8 @@ package com.databuild.examples.basic_graph; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.JsonNode; +import java.util.ArrayList; +import java.util.List; import java.io.File; import java.util.Arrays; import java.util.Collections; @@ -17,6 +19,7 @@ public class GenerateConfigure { System.err.println("Error: Partition ref is required"); System.exit(1); } + List configList = new ArrayList<>(); // Process each partition ref from input arguments Arrays.stream(args).forEach(partitionRef -> { @@ -25,18 +28,19 @@ public class GenerateConfigure { config.outputs = Collections.singletonList(partitionRef); config.args = Arrays.asList(partitionRef); // inputs and env are already initialized as empty collections in the constructor - - try { - ObjectMapper mapper = new ObjectMapper(); - - // Convert config to JsonNode and serialize - JsonNode configNode = mapper.valueToTree(config); - String jsonConfig = mapper.writeValueAsString(configNode); - System.out.println(jsonConfig); - } catch (Exception e) { - System.err.println("Error: Failed to validate or serialize config: " + e.getMessage()); - System.exit(1); - } + configList.add(config); }); + + try { + ObjectMapper mapper = new ObjectMapper(); + + // Convert config list to JsonNode and serialize + JsonNode configNode = mapper.valueToTree(configList); + String jsonConfig = mapper.writeValueAsString(configNode); + System.out.println(jsonConfig); + } catch (Exception e) { + System.err.println("Error: Failed to validate or serialize config: " + e.getMessage()); + System.exit(1); + } } } \ No newline at end of file diff --git a/examples/basic_graph/JobConfig.java b/examples/basic_graph/JobConfig.java index 5eb30cf..844ab53 100644 --- a/examples/basic_graph/JobConfig.java +++ b/examples/basic_graph/JobConfig.java @@ -9,7 +9,7 @@ import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility; @JsonAutoDetect(fieldVisibility = Visibility.ANY) public class JobConfig { - public List inputs; + public List inputs; public List outputs; public List args; public Map env; diff --git a/examples/basic_graph/SumConfigure.java b/examples/basic_graph/SumConfigure.java index 5430a72..ad96dc5 100644 --- a/examples/basic_graph/SumConfigure.java +++ b/examples/basic_graph/SumConfigure.java @@ -6,6 +6,9 @@ import com.fasterxml.jackson.databind.JsonNode; import java.io.File; import java.util.Arrays; import java.util.Collections; +import java.util.stream.Collectors; + +import static com.databuild.examples.basic_graph.GenerateExecute.BASE_PATH; /** * Configure class for generating a random number. @@ -19,12 +22,23 @@ public class SumConfigure { } String partitionRef = args[0]; - String[] upstreams = partitionRef.split("_"); + String[] pathParts = partitionRef.split("/"); + String[] upstreams = Arrays.stream(pathParts[pathParts.length - 1].split("_")) + .map(part -> BASE_PATH + "generated_number/" + part) + .toArray(String[]::new); // Create and populate JobConfig object JobConfig config = new JobConfig(); config.outputs = Collections.singletonList(partitionRef); - config.args = Arrays.asList(upstreams); + config.inputs = Arrays.stream(upstreams) + .map(upstream -> { + DataDep dep = new DataDep(); + dep.depType = "materialize"; + dep.ref = upstream; + return dep; + }) + .collect(Collectors.toList()); + config.args = Collections.singletonList(partitionRef); // Create a hashmap for env with {"OUTPUT_REF": "foo"} config.env = Collections.singletonMap("OUTPUT_REF", args[0]); // inputs and env are already initialized as empty collections in the constructor @@ -33,7 +47,7 @@ public class SumConfigure { ObjectMapper mapper = new ObjectMapper(); // Convert config to JsonNode and serialize - JsonNode configNode = mapper.valueToTree(config); + JsonNode configNode = mapper.valueToTree(Collections.singletonList(config)); String jsonConfig = mapper.writeValueAsString(configNode); System.out.println(jsonConfig); } catch (Exception e) { diff --git a/examples/basic_graph/job_lookup.py b/examples/basic_graph/job_lookup.py index 00d0030..ead531e 100644 --- a/examples/basic_graph/job_lookup.py +++ b/examples/basic_graph/job_lookup.py @@ -12,6 +12,7 @@ def main(): # Partition output prefix makes it obvious which job should fulfill for ref in output_refs: + print(ref, file=sys.stderr) body, tail = ref.rsplit("/", 1) if "generated_number" in body: result["//:generate_number_job"].append(ref) diff --git a/examples/basic_graph/test/generate_number_test.sh b/examples/basic_graph/test/generate_number_test.sh index b6c9176..fe1b35b 100755 --- a/examples/basic_graph/test/generate_number_test.sh +++ b/examples/basic_graph/test/generate_number_test.sh @@ -1,14 +1,15 @@ #!/bin/bash +set -e # Test configure generate_number_job.cfg pippin salem sadie # Test run -generate_number_job.cfg pippin | generate_number_job.exec +generate_number_job.cfg pippin | jq -c ".[0]" | generate_number_job.exec + # Validate that contents of pippin is 43 if [[ "$(cat /tmp/databuild/examples/basic_graph/generated_number/pippin)" != "43" ]]; then echo "Assertion failed: File does not contain 43" cat /tmp/databuild/examples/basic_graph/generated_number/pippin exit 1 fi - diff --git a/examples/basic_graph/test/graph_test.sh b/examples/basic_graph/test/graph_test.sh index 6e02e78..221cd54 100755 --- a/examples/basic_graph/test/graph_test.sh +++ b/examples/basic_graph/test/graph_test.sh @@ -1,4 +1,5 @@ #!/usr/bin/env bash +set -e basic_graph.lookup /tmp/databuild/examples/basic_graph/generated_number/pippin_salem_sadie diff --git a/examples/basic_graph/test/sum_test.sh b/examples/basic_graph/test/sum_test.sh index 3ae6735..afbc81c 100755 --- a/examples/basic_graph/test/sum_test.sh +++ b/examples/basic_graph/test/sum_test.sh @@ -1,4 +1,5 @@ #!/bin/bash +set -e # Test configure sum_job.cfg pippin_salem_sadie @@ -7,11 +8,12 @@ sum_job.cfg pippin_salem_sadie echo -n 43 > /tmp/databuild/examples/basic_graph/generated_number/pippin echo -n 56 > /tmp/databuild/examples/basic_graph/generated_number/salem echo -n 40 > /tmp/databuild/examples/basic_graph/generated_number/sadie -sum_job.cfg pippin_salem_sadie | sum_job.exec + +sum_job.cfg pippin_salem_sadie | jq -c ".[0]" | sum_job.exec + # Validate that contents of pippin is 43 if [[ "$(cat /tmp/databuild/examples/basic_graph/pippin_salem_sadie)" != "139" ]]; then echo "Assertion failed: File does not contain 139" cat /tmp/databuild/examples/basic_graph/sum/pippin_salem_sadie exit 1 fi - diff --git a/examples/hello_world/configure.sh b/examples/hello_world/configure.sh index 727014c..342c366 100755 --- a/examples/hello_world/configure.sh +++ b/examples/hello_world/configure.sh @@ -1,2 +1,2 @@ # Create a test job config -echo "{\"outputs\":[\"$1\"],\"inputs\":[],\"args\":[\"will\", \"build\", \"$1\"],\"env\":{\"foo\":\"bar\"}}" +echo "[{\"outputs\":[\"$1\"],\"inputs\":[],\"args\":[\"will\", \"build\", \"$1\"],\"env\":{\"foo\":\"bar\"}}]" diff --git a/examples/hello_world/test/test.sh b/examples/hello_world/test/test.sh index 05c6b3f..7b98166 100755 --- a/examples/hello_world/test/test.sh +++ b/examples/hello_world/test/test.sh @@ -2,4 +2,4 @@ test_job.cfg nice -test_job.cfg cool | test_job.exec +test_job.cfg cool | jq -c ".[0]" | test_job.exec diff --git a/job/execute_wrapper.sh.tpl b/job/execute_wrapper.sh.tpl index 9c4a3e8..63e9ff0 100755 --- a/job/execute_wrapper.sh.tpl +++ b/job/execute_wrapper.sh.tpl @@ -17,6 +17,23 @@ if [[ -z "$CONFIG_FILE" ]] || [[ "$CONFIG_FILE" == "-" ]]; then trap 'rm -f "$TMP_CONFIG"' EXIT fi +# Use jq to validate the config file +# First check if the file starts with { and ends with } +if [[ $(head -c 1 "$CONFIG_FILE") != "{" ]] || [[ $(tail -c 2 "$CONFIG_FILE" | head -c 1) != "}" ]]; then + echo "The config file must be a non-empty JSON object:" + cat $CONFIG_FILE + exit 1 +fi + +# Then validate that it parses +if ! $JQ 'type == "object"' $CONFIG_FILE > /dev/null 2>&1; then + echo "The config file must be a non-empty JSON object:" + cat $CONFIG_FILE + exit 1 +fi + +# Should be a single JSON object + # Extract and set environment variables from the config eval "$("$JQ" -r '.env | to_entries | .[] | "export " + .key + "=\"" + .value + "\""' "$CONFIG_FILE")"