diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/ContentExclusionUtil.java b/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/ContentExclusionUtil.java index 0510cc0b0..a2d5c4fb8 100644 --- a/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/ContentExclusionUtil.java +++ b/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/ContentExclusionUtil.java @@ -8,8 +8,27 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; +import com.marklogic.client.impl.XmlFactories; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.w3c.dom.Document; +import org.w3c.dom.Node; +import org.w3c.dom.NodeList; + +import javax.xml.namespace.QName; +import javax.xml.parsers.DocumentBuilder; +import javax.xml.transform.OutputKeys; +import javax.xml.transform.Transformer; +import javax.xml.transform.TransformerException; +import javax.xml.transform.dom.DOMSource; +import javax.xml.transform.stream.StreamResult; +import javax.xml.xpath.XPath; +import javax.xml.xpath.XPathConstants; +import javax.xml.xpath.XPathExpression; +import javax.xml.xpath.XPathExpressionException; +import java.io.ByteArrayInputStream; +import java.io.StringWriter; +import java.nio.charset.StandardCharsets; /** * Utility class for applying content exclusions to documents before hash calculation. @@ -17,7 +36,7 @@ * * @since 8.1.0 */ -public class ContentExclusionUtil { +class ContentExclusionUtil { private static final Logger logger = LoggerFactory.getLogger(ContentExclusionUtil.class); private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); @@ -31,7 +50,7 @@ public class ContentExclusionUtil { * @return the modified JSON content with specified paths removed * @throws JsonProcessingException if the JSON content cannot be parsed or serialized */ - public static String applyJsonExclusions(String uri, String jsonContent, String[] jsonPointers) throws JsonProcessingException { + static String applyJsonExclusions(String uri, String jsonContent, String[] jsonPointers) throws JsonProcessingException { if (jsonPointers == null || jsonPointers.length == 0) { return jsonContent; } @@ -72,6 +91,59 @@ private static void removeNodeAtPointer(String uri, JsonNode rootNode, String js } } - // Future method for XML exclusions - // public static String applyXmlExclusions(String xmlContent, String[] xpaths) { ... } + /** + * Applies XPath exclusions to XML content by removing the specified elements. + * + * @param uri the document URI (used for logging purposes) + * @param xmlContent the XML content as a string + * @param xpathExpressions array of XPath expressions identifying elements to exclude + * @return the modified XML content with specified elements removed + * @throws Exception if the XML content cannot be parsed or serialized + */ + static String applyXmlExclusions(String uri, String xmlContent, String... xpathExpressions) throws Exception { + if (xpathExpressions == null || xpathExpressions.length == 0) { + return xmlContent; + } + + DocumentBuilder builder = XmlFactories.getDocumentBuilderFactory().newDocumentBuilder(); + Document document = builder.parse(new ByteArrayInputStream(xmlContent.getBytes(StandardCharsets.UTF_8))); + applyXmlExclusions(uri, document, xpathExpressions); + return serializeDocument(document); + } + + private static void applyXmlExclusions(String uri, Document document, String[] xpathExpressions) { + final XPath xpath = XmlFactories.getXPathFactory().newXPath(); + for (String xpathExpression : xpathExpressions) { + try { + XPathExpression expr = xpath.compile(xpathExpression); + QName returnType = XPathConstants.NODESET; + NodeList nodes = (NodeList) expr.evaluate(document, returnType); + + if (nodes.getLength() == 0) { + logger.debug("XPath '{}' does not match any nodes in document {}, skipping", xpathExpression, uri); + continue; + } + + // Remove nodes in reverse order to avoid index issues + for (int i = nodes.getLength() - 1; i >= 0; i--) { + Node node = nodes.item(i); + Node parent = node.getParentNode(); + if (parent != null) { + parent.removeChild(node); + } + } + } catch (XPathExpressionException e) { + logger.warn("Invalid XPath expression '{}' for document {}: {}", xpathExpression, uri, e.getMessage()); + } + } + } + + private static String serializeDocument(Document document) throws TransformerException { + Transformer transformer = XmlFactories.newTransformer(); + transformer.setOutputProperty(OutputKeys.OMIT_XML_DECLARATION, "yes"); + transformer.setOutputProperty(OutputKeys.INDENT, "no"); + StringWriter writer = new StringWriter(); + transformer.transform(new DOMSource(document), new StreamResult(writer)); + return writer.toString(); + } } diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/IncrementalWriteEvalFilter.java b/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/IncrementalWriteEvalFilter.java index 54343d80e..c48d95273 100644 --- a/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/IncrementalWriteEvalFilter.java +++ b/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/IncrementalWriteEvalFilter.java @@ -31,8 +31,8 @@ class IncrementalWriteEvalFilter extends IncrementalWriteFilter { """; IncrementalWriteEvalFilter(String hashKeyName, String timestampKeyName, boolean canonicalizeJson, - Consumer skippedDocumentsConsumer, String[] jsonExclusions) { - super(hashKeyName, timestampKeyName, canonicalizeJson, skippedDocumentsConsumer, jsonExclusions); + Consumer skippedDocumentsConsumer, String[] jsonExclusions, String[] xmlExclusions) { + super(hashKeyName, timestampKeyName, canonicalizeJson, skippedDocumentsConsumer, jsonExclusions, xmlExclusions); } @Override diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/IncrementalWriteFilter.java b/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/IncrementalWriteFilter.java index 86cc14e62..46dd06cd8 100644 --- a/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/IncrementalWriteFilter.java +++ b/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/IncrementalWriteFilter.java @@ -46,6 +46,7 @@ public static class Builder { private boolean useEvalQuery = false; private Consumer skippedDocumentsConsumer; private String[] jsonExclusions; + private String[] xmlExclusions; /** * @param keyName the name of the MarkLogic metadata key that will hold the hash value; defaults to "incrementalWriteHash". @@ -103,11 +104,20 @@ public Builder jsonExclusions(String... jsonPointers) { return this; } + /** + * @param xpathExpressions XPath expressions identifying XML elements to exclude from hash calculation. + * For example, "//timestamp" or "//metadata/lastModified". + */ + public Builder xmlExclusions(String... xpathExpressions) { + this.xmlExclusions = xpathExpressions; + return this; + } + public IncrementalWriteFilter build() { if (useEvalQuery) { - return new IncrementalWriteEvalFilter(hashKeyName, timestampKeyName, canonicalizeJson, skippedDocumentsConsumer, jsonExclusions); + return new IncrementalWriteEvalFilter(hashKeyName, timestampKeyName, canonicalizeJson, skippedDocumentsConsumer, jsonExclusions, xmlExclusions); } - return new IncrementalWriteOpticFilter(hashKeyName, timestampKeyName, canonicalizeJson, skippedDocumentsConsumer, jsonExclusions); + return new IncrementalWriteOpticFilter(hashKeyName, timestampKeyName, canonicalizeJson, skippedDocumentsConsumer, jsonExclusions, xmlExclusions); } } @@ -116,17 +126,19 @@ public IncrementalWriteFilter build() { private final boolean canonicalizeJson; private final Consumer skippedDocumentsConsumer; private final String[] jsonExclusions; + private final String[] xmlExclusions; // Hardcoding this for now, with a good general purpose hashing function. // See https://xxhash.com for benchmarks. private final LongHashFunction hashFunction = LongHashFunction.xx3(); - public IncrementalWriteFilter(String hashKeyName, String timestampKeyName, boolean canonicalizeJson, Consumer skippedDocumentsConsumer, String[] jsonExclusions) { + public IncrementalWriteFilter(String hashKeyName, String timestampKeyName, boolean canonicalizeJson, Consumer skippedDocumentsConsumer, String[] jsonExclusions, String[] xmlExclusions) { this.hashKeyName = hashKeyName; this.timestampKeyName = timestampKeyName; this.canonicalizeJson = canonicalizeJson; this.skippedDocumentsConsumer = skippedDocumentsConsumer; this.jsonExclusions = jsonExclusions; + this.xmlExclusions = xmlExclusions; } protected final DocumentWriteSet filterDocuments(Context context, Function hashRetriever) { @@ -178,7 +190,6 @@ private String serializeContent(DocumentWriteOperation doc) { JsonCanonicalizer jc; try { if (jsonExclusions != null && jsonExclusions.length > 0) { - // TBD on error handling here, want to get XML supported first. content = ContentExclusionUtil.applyJsonExclusions(doc.getUri(), content, jsonExclusions); } jc = new JsonCanonicalizer(content); @@ -190,6 +201,13 @@ private String serializeContent(DocumentWriteOperation doc) { logger.warn("Unable to canonicalize JSON content for URI {}, using original content for hashing; cause: {}", doc.getUri(), e.getMessage()); } + } else if (xmlExclusions != null && xmlExclusions.length > 0) { + try { + content = ContentExclusionUtil.applyXmlExclusions(doc.getUri(), content, xmlExclusions); + } catch (Exception e) { + logger.warn("Unable to apply XML exclusions for URI {}, using original content for hashing; cause: {}", + doc.getUri(), e.getMessage()); + } } return content; diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/IncrementalWriteOpticFilter.java b/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/IncrementalWriteOpticFilter.java index 3cb8f44e0..d760f3ab4 100644 --- a/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/IncrementalWriteOpticFilter.java +++ b/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/IncrementalWriteOpticFilter.java @@ -20,8 +20,8 @@ class IncrementalWriteOpticFilter extends IncrementalWriteFilter { IncrementalWriteOpticFilter(String hashKeyName, String timestampKeyName, boolean canonicalizeJson, - Consumer skippedDocumentsConsumer, String[] jsonExclusions) { - super(hashKeyName, timestampKeyName, canonicalizeJson, skippedDocumentsConsumer, jsonExclusions); + Consumer skippedDocumentsConsumer, String[] jsonExclusions, String[] xmlExclusions) { + super(hashKeyName, timestampKeyName, canonicalizeJson, skippedDocumentsConsumer, jsonExclusions, xmlExclusions); } @Override diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/impl/XmlFactories.java b/marklogic-client-api/src/main/java/com/marklogic/client/impl/XmlFactories.java index 6773ef19f..5f931dfba 100644 --- a/marklogic-client-api/src/main/java/com/marklogic/client/impl/XmlFactories.java +++ b/marklogic-client-api/src/main/java/com/marklogic/client/impl/XmlFactories.java @@ -12,8 +12,10 @@ import javax.xml.stream.FactoryConfigurationError; import javax.xml.stream.XMLInputFactory; import javax.xml.stream.XMLOutputFactory; +import javax.xml.transform.Transformer; import javax.xml.transform.TransformerConfigurationException; import javax.xml.transform.TransformerFactory; +import javax.xml.xpath.XPathFactory; import java.lang.ref.SoftReference; import java.util.function.Supplier; @@ -27,6 +29,12 @@ public final class XmlFactories { private static final CachedInstancePerThreadSupplier cachedDocumentBuilderFactory = new CachedInstancePerThreadSupplier<>(XmlFactories::makeNewDocumentBuilderFactory); + private static final CachedInstancePerThreadSupplier cachedXPathFactory = + new CachedInstancePerThreadSupplier<>(XPathFactory::newInstance); + + private static final CachedInstancePerThreadSupplier cachedTransformerFactory = + new CachedInstancePerThreadSupplier<>(XmlFactories::makeNewTransformerFactory); + private XmlFactories() {} // preventing instances of utility class /** @@ -152,6 +160,47 @@ public static XMLOutputFactory getOutputFactory() { return cachedOutputFactory.get(); } + /** + * Returns a shared {@link XPathFactory}. + *

+ * Creating XML factories is potentially a pretty expensive operation. Using a shared instance helps to amortize + * this initialization cost via reuse. + * + * @return a {@link XPathFactory} + * + * @since 8.1.0 + */ + public static XPathFactory getXPathFactory() { + return cachedXPathFactory.get(); + } + + /** + * Returns a shared {@link TransformerFactory} configured with secure defaults. + *

+ * Creating XML factories is potentially a pretty expensive operation. Using a shared instance helps to amortize + * this initialization cost via reuse. + * + * @return a securely configured {@link TransformerFactory} + * + * @since 8.1.0 + */ + public static TransformerFactory getTransformerFactory() { + return cachedTransformerFactory.get(); + } + + /** + * Creates a new {@link Transformer} from the shared {@link TransformerFactory}. + * + * @since 8.1.0 + */ + public static Transformer newTransformer() { + try { + return getTransformerFactory().newTransformer(); + } catch (TransformerConfigurationException e) { + throw new RuntimeException("Unable to create new Transformer from TransformerFactory", e); + } + } + /** * A supplier that caches results per thread. *

diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/io/DOMHandle.java b/marklogic-client-api/src/main/java/com/marklogic/client/io/DOMHandle.java index 90d8c0fc4..a19dd138e 100644 --- a/marklogic-client-api/src/main/java/com/marklogic/client/io/DOMHandle.java +++ b/marklogic-client-api/src/main/java/com/marklogic/client/io/DOMHandle.java @@ -18,7 +18,10 @@ import javax.xml.namespace.QName; import javax.xml.parsers.DocumentBuilderFactory; import javax.xml.parsers.ParserConfigurationException; -import javax.xml.xpath.*; +import javax.xml.xpath.XPath; +import javax.xml.xpath.XPathConstants; +import javax.xml.xpath.XPathExpression; +import javax.xml.xpath.XPathExpressionException; import java.io.*; import java.nio.charset.StandardCharsets; @@ -205,7 +208,7 @@ public void setFactory(DocumentBuilderFactory factory) { */ public XPath getXPathProcessor() { if (xpathProcessor == null) - xpathProcessor = makeXPathProcessorFactory().newXPath(); + xpathProcessor = XmlFactories.getXPathFactory().newXPath(); return xpathProcessor; } /** @@ -216,9 +219,6 @@ public XPath getXPathProcessor() { public void setXPathProcessor(XPath xpathProcessor) { this.xpathProcessor = xpathProcessor; } - protected XPathFactory makeXPathProcessorFactory() { - return XPathFactory.newInstance(); - } /** * Evaluate a string XPath expression against the retrieved document. diff --git a/marklogic-client-api/src/test/java/com/marklogic/client/datamovement/filter/AbstractIncrementalWriteTest.java b/marklogic-client-api/src/test/java/com/marklogic/client/datamovement/filter/AbstractIncrementalWriteTest.java new file mode 100644 index 000000000..6c136f0e5 --- /dev/null +++ b/marklogic-client-api/src/test/java/com/marklogic/client/datamovement/filter/AbstractIncrementalWriteTest.java @@ -0,0 +1,53 @@ +/* + * Copyright (c) 2010-2026 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. + */ +package com.marklogic.client.datamovement.filter; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.marklogic.client.document.DocumentWriteOperation; +import com.marklogic.client.io.DocumentMetadataHandle; +import com.marklogic.client.test.AbstractClientTest; +import com.marklogic.client.test.Common; +import org.junit.jupiter.api.BeforeEach; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; + +abstract class AbstractIncrementalWriteTest extends AbstractClientTest { + + static final DocumentMetadataHandle METADATA = new DocumentMetadataHandle() + .withCollections("incremental-test") + .withPermission("rest-reader", DocumentMetadataHandle.Capability.READ, DocumentMetadataHandle.Capability.UPDATE); + + AtomicInteger writtenCount = new AtomicInteger(); + AtomicInteger skippedCount = new AtomicInteger(); + AtomicReference batchFailure = new AtomicReference<>(); + ObjectMapper objectMapper = new ObjectMapper(); + + List docs = new ArrayList<>(); + IncrementalWriteFilter filter; + + @BeforeEach + void setup() { + // Need a user with eval privileges so that the eval filter can be tested. + Common.client = Common.newEvalClient(); + + // Default filter implementation, should be suitable for most tests. + filter = IncrementalWriteFilter.newBuilder() + .onDocumentsSkipped(docs -> skippedCount.addAndGet(docs.length)) + .build(); + } + + final void writeDocs(List docs) { + new WriteBatcherTemplate(Common.client).runWriteJob( + writeBatcher -> writeBatcher + .withDocumentWriteSetFilter(filter) + .onBatchSuccess(batch -> writtenCount.addAndGet(batch.getItems().length)) + .onBatchFailure((batch, failure) -> batchFailure.set(failure)), + + writeBatcher -> docs.forEach(writeBatcher::add) + ); + } +} diff --git a/marklogic-client-api/src/test/java/com/marklogic/client/datamovement/filter/ApplyExclusionsToIncrementalWriteTest.java b/marklogic-client-api/src/test/java/com/marklogic/client/datamovement/filter/ApplyExclusionsToIncrementalWriteTest.java new file mode 100644 index 000000000..19f5dd339 --- /dev/null +++ b/marklogic-client-api/src/test/java/com/marklogic/client/datamovement/filter/ApplyExclusionsToIncrementalWriteTest.java @@ -0,0 +1,142 @@ +/* + * Copyright (c) 2010-2026 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. + */ +package com.marklogic.client.datamovement.filter; + +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.marklogic.client.impl.DocumentWriteOperationImpl; +import com.marklogic.client.io.Format; +import com.marklogic.client.io.JacksonHandle; +import com.marklogic.client.io.StringHandle; +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +class ApplyExclusionsToIncrementalWriteTest extends AbstractIncrementalWriteTest { + + @Test + void jsonExclusions() { + filter = IncrementalWriteFilter.newBuilder() + .jsonExclusions("/timestamp", "/metadata/lastModified") + .onDocumentsSkipped(docs -> skippedCount.addAndGet(docs.length)) + .build(); + + // Write initial documents with three keys + docs = new ArrayList<>(); + for (int i = 1; i <= 5; i++) { + ObjectNode doc = objectMapper.createObjectNode(); + doc.put("id", i); + doc.put("name", "Document " + i); + doc.put("timestamp", "2025-01-01T10:00:00Z"); + doc.putObject("metadata") + .put("lastModified", "2025-01-01T10:00:00Z") + .put("author", "Test User"); + docs.add(new DocumentWriteOperationImpl("/incremental/test/json-doc-" + i + ".json", METADATA, new JacksonHandle(doc))); + } + + writeDocs(docs); + assertEquals(5, writtenCount.get()); + assertEquals(0, skippedCount.get()); + + // Write again with different values for excluded fields - should be skipped + docs = new ArrayList<>(); + for (int i = 1; i <= 5; i++) { + ObjectNode doc = objectMapper.createObjectNode(); + doc.put("id", i); + doc.put("name", "Document " + i); + doc.put("timestamp", "2026-01-02T15:30:00Z"); // Changed + doc.putObject("metadata") + .put("lastModified", "2026-01-02T15:30:00Z") // Changed + .put("author", "Test User"); + docs.add(new DocumentWriteOperationImpl("/incremental/test/json-doc-" + i + ".json", METADATA, new JacksonHandle(doc))); + } + + writeDocs(docs); + assertEquals(5, writtenCount.get(), "Documents should be skipped since only excluded fields changed"); + assertEquals(5, skippedCount.get()); + + // Write again with actual content change - should NOT be skipped + docs = new ArrayList<>(); + for (int i = 1; i <= 5; i++) { + ObjectNode doc = objectMapper.createObjectNode(); + doc.put("id", i); + doc.put("name", "Modified Document " + i); // Changed + doc.put("timestamp", "2026-01-02T16:00:00Z"); + doc.putObject("metadata") + .put("lastModified", "2026-01-02T16:00:00Z") + .put("author", "Test User"); + docs.add(new DocumentWriteOperationImpl("/incremental/test/json-doc-" + i + ".json", METADATA, new JacksonHandle(doc))); + } + + writeDocs(docs); + assertEquals(10, writtenCount.get(), "Documents should be written since non-excluded content changed"); + assertEquals(5, skippedCount.get(), "Skip count should remain at 5"); + } + + @Test + void xmlExclusions() { + filter = IncrementalWriteFilter.newBuilder() + .xmlExclusions("//timestamp", "//metadata/lastModified") + .onDocumentsSkipped(docs -> skippedCount.addAndGet(docs.length)) + .build(); + + // Write initial documents + docs = new ArrayList<>(); + for (int i = 1; i <= 5; i++) { + String xml = "" + + "" + i + "" + + "Document " + i + "" + + "2025-01-01T10:00:00Z" + + "" + + "Test User" + + "2025-01-01T10:00:00Z" + + "" + + ""; + docs.add(new DocumentWriteOperationImpl("/incremental/test/xml-doc-" + i + ".xml", METADATA, new StringHandle(xml).withFormat(Format.XML))); + } + + writeDocs(docs); + assertEquals(5, writtenCount.get()); + assertEquals(0, skippedCount.get()); + + // Write again with different values for excluded fields - should be skipped + docs = new ArrayList<>(); + for (int i = 1; i <= 5; i++) { + String xml = "" + + "" + i + "" + + "Document " + i + "" + + "2026-01-02T15:30:00Z" + // Changed + "" + + "Test User" + + "2026-01-02T15:30:00Z" + // Changed + "" + + ""; + docs.add(new DocumentWriteOperationImpl("/incremental/test/xml-doc-" + i + ".xml", METADATA, new StringHandle(xml).withFormat(Format.XML))); + } + + writeDocs(docs); + assertEquals(5, writtenCount.get(), "Documents should be skipped since only excluded fields changed"); + assertEquals(5, skippedCount.get()); + + // Write again with actual content change - should NOT be skipped + docs = new ArrayList<>(); + for (int i = 1; i <= 5; i++) { + String xml = "" + + "" + i + "" + + "Modified Document " + i + "" + // Changed + "2026-01-02T16:00:00Z" + + "" + + "Test User" + + "2026-01-02T16:00:00Z" + + "" + + ""; + docs.add(new DocumentWriteOperationImpl("/incremental/test/xml-doc-" + i + ".xml", METADATA, new StringHandle(xml).withFormat(Format.XML))); + } + + writeDocs(docs); + assertEquals(10, writtenCount.get(), "Documents should be written since non-excluded content changed"); + assertEquals(5, skippedCount.get(), "Skip count should remain at 5"); + } +} diff --git a/marklogic-client-api/src/test/java/com/marklogic/client/datamovement/filter/IncrementalWriteTest.java b/marklogic-client-api/src/test/java/com/marklogic/client/datamovement/filter/IncrementalWriteTest.java index 9929fdc98..723296b72 100644 --- a/marklogic-client-api/src/test/java/com/marklogic/client/datamovement/filter/IncrementalWriteTest.java +++ b/marklogic-client-api/src/test/java/com/marklogic/client/datamovement/filter/IncrementalWriteTest.java @@ -3,7 +3,6 @@ */ package com.marklogic.client.datamovement.filter; -import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; import com.marklogic.client.document.*; import com.marklogic.client.impl.DocumentWriteOperationImpl; @@ -11,42 +10,15 @@ import com.marklogic.client.io.Format; import com.marklogic.client.io.JacksonHandle; import com.marklogic.client.io.StringHandle; -import com.marklogic.client.test.AbstractClientTest; import com.marklogic.client.test.Common; -import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import java.util.ArrayList; import java.util.List; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.atomic.AtomicReference; import static org.junit.jupiter.api.Assertions.*; -class IncrementalWriteTest extends AbstractClientTest { - - private static final DocumentMetadataHandle METADATA = new DocumentMetadataHandle() - .withCollections("incremental-test") - .withPermission("rest-reader", DocumentMetadataHandle.Capability.READ, DocumentMetadataHandle.Capability.UPDATE); - - AtomicInteger writtenCount = new AtomicInteger(); - AtomicInteger skippedCount = new AtomicInteger(); - AtomicReference batchFailure = new AtomicReference<>(); - ObjectMapper objectMapper = new ObjectMapper(); - - List docs = new ArrayList<>(); - IncrementalWriteFilter filter; - - @BeforeEach - void setup() { - // Need a user with eval privileges so that the eval filter can be tested. - Common.client = Common.newEvalClient(); - - // Default filter implementation, should be suitable for most tests. - filter = IncrementalWriteFilter.newBuilder() - .onDocumentsSkipped(docs -> skippedCount.addAndGet(docs.length)) - .build(); - } +class IncrementalWriteTest extends AbstractIncrementalWriteTest { @Test void opticFilter() { @@ -218,65 +190,6 @@ void nullIsIgnoredForKeyNames() { assertNotNull(metadata.getMetadataValues().get("incrementalWriteTimestamp")); } - @Test - void jsonExclusions() { - filter = IncrementalWriteFilter.newBuilder() - .jsonExclusions("/timestamp", "/metadata/lastModified") - .onDocumentsSkipped(docs -> skippedCount.addAndGet(docs.length)) - .build(); - - // Write initial documents with three keys - docs = new ArrayList<>(); - for (int i = 1; i <= 5; i++) { - ObjectNode doc = objectMapper.createObjectNode(); - doc.put("id", i); - doc.put("name", "Document " + i); - doc.put("timestamp", "2025-01-01T10:00:00Z"); - doc.putObject("metadata") - .put("lastModified", "2025-01-01T10:00:00Z") - .put("author", "Test User"); - docs.add(new DocumentWriteOperationImpl("/incremental/test/json-doc-" + i + ".json", METADATA, new JacksonHandle(doc))); - } - - writeDocs(docs); - assertEquals(5, writtenCount.get()); - assertEquals(0, skippedCount.get()); - - // Write again with different values for excluded fields - should be skipped - docs = new ArrayList<>(); - for (int i = 1; i <= 5; i++) { - ObjectNode doc = objectMapper.createObjectNode(); - doc.put("id", i); - doc.put("name", "Document " + i); - doc.put("timestamp", "2026-01-02T15:30:00Z"); // Changed - doc.putObject("metadata") - .put("lastModified", "2026-01-02T15:30:00Z") // Changed - .put("author", "Test User"); - docs.add(new DocumentWriteOperationImpl("/incremental/test/json-doc-" + i + ".json", METADATA, new JacksonHandle(doc))); - } - - writeDocs(docs); - assertEquals(5, writtenCount.get(), "Documents should be skipped since only excluded fields changed"); - assertEquals(5, skippedCount.get()); - - // Write again with actual content change - should NOT be skipped - docs = new ArrayList<>(); - for (int i = 1; i <= 5; i++) { - ObjectNode doc = objectMapper.createObjectNode(); - doc.put("id", i); - doc.put("name", "Modified Document " + i); // Changed - doc.put("timestamp", "2026-01-02T16:00:00Z"); - doc.putObject("metadata") - .put("lastModified", "2026-01-02T16:00:00Z") - .put("author", "Test User"); - docs.add(new DocumentWriteOperationImpl("/incremental/test/json-doc-" + i + ".json", METADATA, new JacksonHandle(doc))); - } - - writeDocs(docs); - assertEquals(10, writtenCount.get(), "Documents should be written since non-excluded content changed"); - assertEquals(5, skippedCount.get(), "Skip count should remain at 5"); - } - private void verifyIncrementalWriteWorks() { writeTenDocuments(); verifyDocumentsHasHashInMetadataKey(); @@ -337,15 +250,4 @@ private void modifyFiveDocuments() { } writeDocs(docs); } - - private void writeDocs(List docs) { - new WriteBatcherTemplate(Common.client).runWriteJob( - writeBatcher -> writeBatcher - .withDocumentWriteSetFilter(filter) - .onBatchSuccess(batch -> writtenCount.addAndGet(batch.getItems().length)) - .onBatchFailure((batch, failure) -> batchFailure.set(failure)), - - writeBatcher -> docs.forEach(writeBatcher::add) - ); - } }