From 4e5b792c1fe2e77cb35851a64615ee5e66795065 Mon Sep 17 00:00:00 2001 From: Rob Rudin Date: Mon, 28 Sep 2026 15:56:35 -0400 Subject: [PATCH] PDP-2114 Added --incremental-write-source-uri-name Depends on this PR being merged first, won't work until then - https://github.com/marklogic/marklogic-spark-connector/pull/697 --- flux-cli/build.gradle | 2 +- .../flux/impl/importdata/IncrementalWriteParams.java | 7 +++++++ .../flux/impl/importdata/ImportFilesOptionsTest.java | 2 ++ 3 files changed, 10 insertions(+), 1 deletion(-) diff --git a/flux-cli/build.gradle b/flux-cli/build.gradle index ee09a413..e4fdb6a9 100644 --- a/flux-cli/build.gradle +++ b/flux-cli/build.gradle @@ -31,7 +31,7 @@ dependencies { implementation "info.picocli:picocli:${picocliVersion}" // Need this for compilation as it's not an API dependency of the Spark connector. - compileOnly "com.marklogic:marklogic-client-api:8.1.0" + compileOnly "com.marklogic:marklogic-client-api:8.3-SNAPSHOT" // The shadow jar intended for usage with spark-submit needs the above "core" libraries as well, but not the Spark // API since spark-submit provides that. diff --git a/flux-cli/src/main/java/com/marklogic/flux/impl/importdata/IncrementalWriteParams.java b/flux-cli/src/main/java/com/marklogic/flux/impl/importdata/IncrementalWriteParams.java index e700d74c..cf094963 100644 --- a/flux-cli/src/main/java/com/marklogic/flux/impl/importdata/IncrementalWriteParams.java +++ b/flux-cli/src/main/java/com/marklogic/flux/impl/importdata/IncrementalWriteParams.java @@ -52,6 +52,12 @@ static class SchemaViewGroup { // This is the default in the Java Client as well, but stating here it makes it clear that it's the default. private String hashKeyName = "incrementalWriteHash"; + @CommandLine.Option( + names = "--incremental-write-source-uri-name", + description = "Name of the optional metadata key in which to store the source URI of the document being written. No default value." + ) + private String sourceUriKeyName; + @CommandLine.Option( names = "--incremental-write-timestamp-name", description = "Name of the optional metadata key in which to store the timestamp for when the document was written. No default value." @@ -97,6 +103,7 @@ public Map makeOptions() { Options.WRITE_INCREMENTAL_SCHEMA, schemaView != null ? schemaView.schema : null, Options.WRITE_INCREMENTAL_VIEW, schemaView != null ? schemaView.view : null, Options.WRITE_INCREMENTAL_HASH_KEY_NAME, hashKeyName, + Options.WRITE_INCREMENTAL_SOURCE_URI_KEY_NAME, sourceUriKeyName, Options.WRITE_INCREMENTAL_TIMESTAMP_KEY_NAME, timestampKeyName, Options.WRITE_INCREMENTAL_CANONICALIZE_JSON, dontCanonicalizeJson != null ? (!dontCanonicalizeJson ? "true" : "false") : null, Options.WRITE_INCREMENTAL_JSON_EXCLUSIONS, jsonExclusions.isEmpty() ? null : String.join("\n", jsonExclusions), diff --git a/flux-cli/src/test/java/com/marklogic/flux/impl/importdata/ImportFilesOptionsTest.java b/flux-cli/src/test/java/com/marklogic/flux/impl/importdata/ImportFilesOptionsTest.java index 3988f94c..e0119be3 100644 --- a/flux-cli/src/test/java/com/marklogic/flux/impl/importdata/ImportFilesOptionsTest.java +++ b/flux-cli/src/test/java/com/marklogic/flux/impl/importdata/ImportFilesOptionsTest.java @@ -233,6 +233,7 @@ void incrementalWriteOptions() { "--incremental-write-schema", "mySchema", "--incremental-write-view", "myView", "--incremental-write-hash-name", "customHash", + "--incremental-write-source-uri-name", "mySourceUri", "--incremental-write-timestamp-name", "customTimestamp", "--incremental-write-disable-json-canonicalization", "--incremental-write-json-exclusion", "/json1", @@ -249,6 +250,7 @@ void incrementalWriteOptions() { Options.WRITE_INCREMENTAL_SCHEMA, "mySchema", Options.WRITE_INCREMENTAL_VIEW, "myView", Options.WRITE_INCREMENTAL_HASH_KEY_NAME, "customHash", + Options.WRITE_INCREMENTAL_SOURCE_URI_KEY_NAME, "mySourceUri", Options.WRITE_INCREMENTAL_TIMESTAMP_KEY_NAME, "customTimestamp", Options.WRITE_INCREMENTAL_CANONICALIZE_JSON, "false", Options.WRITE_INCREMENTAL_JSON_EXCLUSIONS, "/json1\n/json2",