From 6bff5ce6586386074e64745995f1e1519a96e743 Mon Sep 17 00:00:00 2001 From: Stuart Axelbrooke Date: Thu, 17 Apr 2025 17:27:55 -0700 Subject: [PATCH] Add sum job --- examples/basic_graph/.bazelrc | 15 +++++++++ examples/basic_graph/BUILD.bazel | 29 +++++++++++++--- examples/basic_graph/Configure.java | 24 -------------- examples/basic_graph/GenerateConfigure.java | 26 +++++++++++++++ .../{Execute.java => GenerateExecute.java} | 10 +++--- examples/basic_graph/SumConfigure.java | 31 +++++++++++++++++ examples/basic_graph/SumExecute.java | 33 +++++++++++++++++++ 7 files changed, 136 insertions(+), 32 deletions(-) delete mode 100644 examples/basic_graph/Configure.java create mode 100644 examples/basic_graph/GenerateConfigure.java rename examples/basic_graph/{Execute.java => GenerateExecute.java} (89%) create mode 100644 examples/basic_graph/SumConfigure.java create mode 100644 examples/basic_graph/SumExecute.java diff --git a/examples/basic_graph/.bazelrc b/examples/basic_graph/.bazelrc index aec09ba..874d6a5 100644 --- a/examples/basic_graph/.bazelrc +++ b/examples/basic_graph/.bazelrc @@ -1,2 +1,17 @@ build --java_runtime_version=21 build --tool_java_runtime_version=21 + +# Default to quiet mode for run commands +run --ui_event_filters=-info,-stdout,-stderr +run --noshow_progress +run --noannounce_rc + +# Explicit quiet mode configuration (same as default) +run:quiet --ui_event_filters=-info,-stdout,-stderr +run:quiet --noshow_progress +run:quiet --noannounce_rc + +# Loud mode configuration (override the quiet default) +run:loud --ui_event_filters= +run:loud --show_progress +run:loud --announce_rc diff --git a/examples/basic_graph/BUILD.bazel b/examples/basic_graph/BUILD.bazel index a774521..eac3964 100644 --- a/examples/basic_graph/BUILD.bazel +++ b/examples/basic_graph/BUILD.bazel @@ -9,12 +9,33 @@ databuild_job( java_binary( name = "generate_number_configure", - srcs = ["Configure.java"], - main_class = "com.databuild.examples.basic_graph.Configure", + srcs = ["GenerateConfigure.java"], + main_class = "com.databuild.examples.basic_graph.GenerateConfigure", ) java_binary( name = "generate_number_execute", - srcs = ["Execute.java"], - main_class = "com.databuild.examples.basic_graph.Execute", + srcs = ["GenerateExecute.java"], + main_class = "com.databuild.examples.basic_graph.GenerateExecute", +) + +databuild_job( + name = "sum_job", + configure = ":sum_configure", + execute = ":sum_execute", +) + +java_binary( + name = "sum_configure", + srcs = ["SumConfigure.java"], + main_class = "com.databuild.examples.basic_graph.SumConfigure", +) + +java_binary( + name = "sum_execute", + srcs = [ + "GenerateExecute.java", + "SumExecute.java", + ], + main_class = "com.databuild.examples.basic_graph.SumExecute", ) diff --git a/examples/basic_graph/Configure.java b/examples/basic_graph/Configure.java deleted file mode 100644 index b0277f3..0000000 --- a/examples/basic_graph/Configure.java +++ /dev/null @@ -1,24 +0,0 @@ -package com.databuild.examples.basic_graph; - -/** - * Configure class for generating a random number. - * This class creates a job configuration for generating a random number based on the partition ref. - */ -public class Configure { - public static void main(String[] args) { - if (args.length < 1) { - System.err.println("Error: Partition ref (output path) is required"); - System.exit(1); - } - - String partitionRef = args[0]; - - // Create a job config for generating a random number - String config = String.format( - "{\"outputs\":[\"%s\"],\"inputs\":[],\"args\":[\"%s\"],\"env\":{}}", - partitionRef, partitionRef - ); - - System.out.println(config); - } -} \ No newline at end of file diff --git a/examples/basic_graph/GenerateConfigure.java b/examples/basic_graph/GenerateConfigure.java new file mode 100644 index 0000000..20560c8 --- /dev/null +++ b/examples/basic_graph/GenerateConfigure.java @@ -0,0 +1,26 @@ +package com.databuild.examples.basic_graph; + +import java.util.Arrays; + +/** + * Configure class for generating a random number. + * This class creates a job configuration for generating a random number based on the partition ref. + */ +public class GenerateConfigure { + public static void main(String[] args) { + if (args.length < 1) { + System.err.println("Error: Partition ref is required"); + System.exit(1); + } + + // Process each partition ref from input arguments + Arrays.stream(args).forEach(partitionRef -> { + // Create a job config for generating a random number + String config = String.format( + "{\"outputs\":[\"%s\"],\"inputs\":[],\"args\":[\"%s\"],\"env\":{}}", + partitionRef, partitionRef + ); + System.out.println(config); + }); + } +} \ No newline at end of file diff --git a/examples/basic_graph/Execute.java b/examples/basic_graph/GenerateExecute.java similarity index 89% rename from examples/basic_graph/Execute.java rename to examples/basic_graph/GenerateExecute.java index 1377718..5d189d8 100644 --- a/examples/basic_graph/Execute.java +++ b/examples/basic_graph/GenerateExecute.java @@ -11,7 +11,9 @@ import java.util.Random; * Execute class for generating a random number. * This class generates a random number based on the partition ref. */ -public class Execute { +public class GenerateExecute { + public static String BASE_PATH = "/tmp/databuild/examples/basic_graph/"; + public static void main(String[] args) { if (args.length < 1) { System.err.println("Error: Partition ref (output path) is required"); @@ -35,7 +37,7 @@ public class Execute { Random random = new Random(seed); // Generate a random number - int randomNumber = random.nextInt(Integer.MAX_VALUE); + int randomNumber = random.nextInt(100); // Write the random number to the output file try (FileWriter writer = new FileWriter(partitionRef)) { @@ -45,11 +47,11 @@ public class Execute { System.out.println("Generated random number " + randomNumber + " for partition " + partitionRef); // Write the random number to the output file - String outputPath = "/tmp/databuild/examples/basic_graph/" + partitionRef; + String outputPath = BASE_PATH + partitionRef; System.out.println("Writing random number " + randomNumber + " to " + outputPath); // Ensure dir exists new java.io.File(outputPath).getParentFile().mkdirs(); - // Write number + // Write number (overwrite) try (FileWriter writer = new FileWriter(outputPath)) { writer.write(String.valueOf(randomNumber)); } diff --git a/examples/basic_graph/SumConfigure.java b/examples/basic_graph/SumConfigure.java new file mode 100644 index 0000000..4f3f24e --- /dev/null +++ b/examples/basic_graph/SumConfigure.java @@ -0,0 +1,31 @@ +package com.databuild.examples.basic_graph; + +import java.util.ArrayList; +import java.util.Arrays; + +/** + * Configure class for generating a random number. + * This class creates a job configuration for generating a random number based on the partition ref. + */ +public class SumConfigure { + public static void main(String[] args) { + if (args.length != 1) { + System.err.println("Error: Must provide exactly one partition ref as an argument"); + System.exit(1); + } + + String partitionRef = args[0]; + String[] upstreams = partitionRef.split("_"); + + // Create a list of quoted upstream values + ArrayList quotedUpstreams = new ArrayList<>(); + Arrays.stream(upstreams).forEach(s -> quotedUpstreams.add("\"" + s + "\"")); + + // Create a job config for generating a random number + String config = String.format( + "{\"outputs\":[\"%s\"],\"inputs\":[],\"args\":[%s],\"env\":{}}", + partitionRef, String.join(",", quotedUpstreams) + ); + System.out.println(config); + } +} \ No newline at end of file diff --git a/examples/basic_graph/SumExecute.java b/examples/basic_graph/SumExecute.java new file mode 100644 index 0000000..6b0d8d3 --- /dev/null +++ b/examples/basic_graph/SumExecute.java @@ -0,0 +1,33 @@ +package com.databuild.examples.basic_graph; + +import static com.databuild.examples.basic_graph.GenerateExecute.BASE_PATH; + +public class SumExecute { + public static void main(String[] args) { + if (args.length < 1) { + System.err.println("Error: Partition ref (output path) is required"); + System.exit(1); + } + + // For each arg, load it from the file system and add it to the sum + int sum = 0; + for (String partitionRef : args) { + try { + String path = BASE_PATH + partitionRef; + int partitionValue = Integer.parseInt(new String(java.nio.file.Files.readAllBytes(java.nio.file.Paths.get(path)))); + System.out.println("Summing partition " + partitionRef + " with value " + partitionValue); + sum += partitionValue; + } catch (Exception e) { + System.err.println("Error: Failed to read partition " + partitionRef + ": " + e.getMessage()); + e.printStackTrace(); + } + } + System.out.println("Sum of " + args.length + " partitions: " + sum); + // Write the sum to the output file + try (java.io.FileWriter writer = new java.io.FileWriter(BASE_PATH + args[0])) { + writer.write(sum); + } catch (Exception e) { + System.err.println("Error: Failed to write sum to " + args[0] + ": " + e.getMessage()); + } + } +}