diff --git a/marklogic-client-api-functionaltests/src/test/java/com/marklogic/client/fastfunctest/AbstractFunctionalTest.java b/marklogic-client-api-functionaltests/src/test/java/com/marklogic/client/fastfunctest/AbstractFunctionalTest.java index 8822634ff..28c62a833 100644 --- a/marklogic-client-api-functionaltests/src/test/java/com/marklogic/client/fastfunctest/AbstractFunctionalTest.java +++ b/marklogic-client-api-functionaltests/src/test/java/com/marklogic/client/fastfunctest/AbstractFunctionalTest.java @@ -29,8 +29,6 @@ */ public abstract class AbstractFunctionalTest extends BasicJavaClientREST { - protected final static String OPTIC_USER = "opticUser"; - protected final static String OPTIC_USER_PASSWORD = "0pt1c"; protected final static String DB_NAME = "java-functest"; protected static DatabaseClient client; diff --git a/marklogic-client-api-functionaltests/src/test/java/com/marklogic/client/datamovement/functionaltests/ApplyTransformTest.java b/marklogic-client-api-functionaltests/src/test/java/com/marklogic/client/fastfunctest/ApplyTransformTest.java similarity index 99% rename from marklogic-client-api-functionaltests/src/test/java/com/marklogic/client/datamovement/functionaltests/ApplyTransformTest.java rename to marklogic-client-api-functionaltests/src/test/java/com/marklogic/client/fastfunctest/ApplyTransformTest.java index 6d6c8f473..823852b11 100644 --- a/marklogic-client-api-functionaltests/src/test/java/com/marklogic/client/datamovement/functionaltests/ApplyTransformTest.java +++ b/marklogic-client-api-functionaltests/src/test/java/com/marklogic/client/fastfunctest/ApplyTransformTest.java @@ -14,7 +14,7 @@ * limitations under the License. */ -package com.marklogic.client.datamovement.functionaltests; +package com.marklogic.client.fastfunctest; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; @@ -23,6 +23,7 @@ import com.marklogic.client.admin.TransformExtensionsManager; import com.marklogic.client.datamovement.*; import com.marklogic.client.datamovement.ApplyTransformListener.ApplyResult; +import com.marklogic.client.datamovement.functionaltests.WriteHostBatcherTest; import com.marklogic.client.document.DocumentPage; import com.marklogic.client.document.DocumentRecord; import com.marklogic.client.document.ServerTransform; diff --git a/marklogic-client-api-functionaltests/src/test/java/com/marklogic/client/datamovement/functionaltests/DeleteListenerTest.java b/marklogic-client-api-functionaltests/src/test/java/com/marklogic/client/fastfunctest/DeleteListenerTest.java similarity index 66% rename from marklogic-client-api-functionaltests/src/test/java/com/marklogic/client/datamovement/functionaltests/DeleteListenerTest.java rename to marklogic-client-api-functionaltests/src/test/java/com/marklogic/client/fastfunctest/DeleteListenerTest.java index d4aa0bc4c..bd9933b10 100644 --- a/marklogic-client-api-functionaltests/src/test/java/com/marklogic/client/datamovement/functionaltests/DeleteListenerTest.java +++ b/marklogic-client-api-functionaltests/src/test/java/com/marklogic/client/fastfunctest/DeleteListenerTest.java @@ -13,37 +13,54 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.marklogic.client.datamovement.functionaltests; +package com.marklogic.client.fastfunctest; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.marklogic.client.DatabaseClient; -import com.marklogic.client.datamovement.*; +import com.marklogic.client.datamovement.DataMovementManager; +import com.marklogic.client.datamovement.DeleteListener; +import com.marklogic.client.datamovement.JobTicket; +import com.marklogic.client.datamovement.QueryBatcher; +import com.marklogic.client.datamovement.UrisToWriterListener; +import com.marklogic.client.datamovement.WriteBatcher; +import com.marklogic.client.datamovement.functionaltests.WriteHostBatcherTest; import com.marklogic.client.document.DocumentPage; import com.marklogic.client.document.DocumentRecord; -import com.marklogic.client.functionaltest.BasicJavaClientREST; -import com.marklogic.client.io.*; +import com.marklogic.client.io.DocumentMetadataHandle; +import com.marklogic.client.io.FileHandle; +import com.marklogic.client.io.Format; +import com.marklogic.client.io.JacksonHandle; +import com.marklogic.client.io.StringHandle; import com.marklogic.client.query.StructuredQueryBuilder; import org.apache.commons.io.FileUtils; -import org.junit.jupiter.api.*; - -import java.io.*; -import java.util.*; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileReader; +import java.io.FileWriter; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.Iterator; +import java.util.Set; import java.util.concurrent.atomic.AtomicInteger; -import static org.junit.jupiter.api.Assertions.*; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; -public class DeleteListenerTest extends BasicJavaClientREST { +public class DeleteListenerTest extends AbstractFunctionalTest { - private static String dbName = "DeleteListener"; private static DataMovementManager dmManager = null; private static final String TEST_DIR_PREFIX = "/WriteHostBatcher-testdata/"; private static DatabaseClient dbClient; - private static String user = "admin"; - private static int port = 8000; - private static String password = "admin"; - private static String server = "App-Services"; private static JacksonHandle jacksonHandle; private static StringHandle stringHandle; private static FileHandle fileHandle; @@ -54,37 +71,10 @@ public class DeleteListenerTest extends BasicJavaClientREST { private static File fileJson; private static JsonNode jsonNode; private static final String query1 = "fn:count(fn:doc())"; - private static String[] hostNames; - private static int forestCount = 1; @BeforeAll public static void setUpBeforeClass() throws Exception { - loadGradleProperties(); - server = getRestAppServerName(); - port = getRestAppServerPort(); - - hostNames = getHosts(); - createDB(dbName); - Thread.currentThread().sleep(500L); - //Ensure DB has at-least one forest - createForestonHost(dbName + "-" + forestCount, dbName, hostNames[0]); - forestCount++; - for (String forestHost : hostNames) { - for(int i = 0; i < new Random().nextInt(3); i++) { - createForestonHost(dbName + "-" + forestCount, dbName, forestHost); - forestCount++; - } - Thread.currentThread().sleep(500L); - } - // Create App Server if needed. - createRESTServerWithDB(server, port); - - assocRESTServer(server, dbName, port); - if (IsSecurityEnabled()) { - enableSecurityOnRESTServer(server, dbName); - } - - dbClient = getDatabaseClient(user, password, getConnType()); + dbClient = newDatabaseClientBuilder().build(); dmManager = dbClient.newDataMovementManager(); // JacksonHandle @@ -105,21 +95,10 @@ public static void setUpBeforeClass() throws Exception { fileHandle.setFormat(Format.JSON); } - @AfterAll - public static void tearDownAfterClass() throws Exception { - associateRESTServerWithDB(server, "Documents"); - for (int i = 0; i < forestCount -1; i++) { - System.out.println(dbName + "-" + (i + 1)); - detachForest(dbName, dbName + "-" + (i + 1)); - deleteForest(dbName + "-" + (i + 1)); - } - - deleteDB(dbName); - } - @BeforeEach public void setUp() throws Exception { - Thread.currentThread().sleep(1000L); + deleteDocuments(client); + WriteBatcher ihb2 = dmManager.newWriteBatcher(); ihb2.withBatchSize(27).withThreadCount(10); dmManager.startJob(ihb2); @@ -132,11 +111,6 @@ public void setUp() throws Exception { assertEquals(2000, dbClient.newServerEval().xquery(query1).eval().next().getNumber().intValue()); } - @AfterEach - public void tearDown() throws Exception { - clearDB(port); - } - @Test public void massDeleteSingleThread() throws Exception { HashSet urisList = new HashSet<>(); @@ -162,7 +136,6 @@ public void massDeleteSingleThread() throws Exception { queryBatcher.awaitCompletion(); dmManager.stopJob(ticket); - Thread.currentThread().sleep(2000L); assertEquals(2000, urisList.size()); AtomicInteger successDocs = new AtomicInteger(); @@ -241,32 +214,31 @@ public void massDeleteMultipleThreads() throws Exception { } @Test - public void massDeleteConsistentSnapShot() throws Exception { - Map props = new HashMap(); - props.put("merge-timestamp", "-6000000000"); - changeProperty(props, "/manage/v2/databases/" + dbName + "/properties"); - Thread.currentThread().sleep(5000L); - - QueryBatcher queryBatcher = dmManager.newQueryBatcher( - new StructuredQueryBuilder().collection("DeleteListener")) - .withBatchSize(7) - .withConsistentSnapshot() - .withThreadCount(5) - .onUrisReady(new DeleteListener()) - .onQueryFailure(throwable -> { - System.out.println("Exceptions thrown from callback onQueryFailure"); - throwable.printStackTrace(); - - }); + public void massDeleteConsistentSnapShot() { + setMergeTimestamp(DB_NAME, "-600000000"); + + try { + QueryBatcher queryBatcher = dmManager.newQueryBatcher( + new StructuredQueryBuilder().collection("DeleteListener")) + .withBatchSize(7) + .withConsistentSnapshot() + .withThreadCount(5) + .onUrisReady(new DeleteListener()) + .onQueryFailure(throwable -> { + System.out.println("Exceptions thrown from callback onQueryFailure"); + throwable.printStackTrace(); + + }); + + JobTicket ticket = dmManager.startJob(queryBatcher); + queryBatcher.awaitCompletion(); + dmManager.stopJob(ticket); + + assertEquals(0, dbClient.newServerEval().xquery(query1).eval().next().getNumber().intValue()); + } finally { + setMergeTimestamp(DB_NAME, "0"); + } - JobTicket ticket = dmManager.startJob(queryBatcher); - queryBatcher.awaitCompletion(); - dmManager.stopJob(ticket); - - props.put("merge-timestamp", "0"); - changeProperty(props, "/manage/v2/databases/" + dbName + "/properties"); - // if ( failures2.length() > 0 ) fail(failures2.toString()); - assertEquals(0, dbClient.newServerEval().xquery(query1).eval().next().getNumber().intValue()); } @Test @@ -303,84 +275,85 @@ public void deleteNonExistentDoc() throws Exception { // ISSUE 94 @Test public void deleteServerFile() throws Exception { - - Map props = new HashMap(); - props.put("merge-timestamp", "-6000000000"); - changeProperty(props, "/manage/v2/databases/" + dbName + "/properties"); - Thread.currentThread().sleep(5000L); - - class MyRunnable implements Runnable { - @Override - public void run() { - for (int j = 1999; j >= 200; j--) { - dbClient.newDocumentManager().delete("/local/json-" + j); - } - } - } - Thread t1; - t1 = new Thread(new MyRunnable()); - - Set urisList = Collections.synchronizedSet(new HashSet<>()); - - QueryBatcher queryBatcher = dmManager.newQueryBatcher( - new StructuredQueryBuilder().collection("DeleteListener")) - .withBatchSize(11) - .withThreadCount(4) - .withConsistentSnapshot() - .onUrisReady(batch -> { - for (String s : batch.getItems()) { - urisList.add(s); - } - }) - .onQueryFailure(throwable -> { - System.out.println("Exceptions thrown from callback onQueryFailure"); - throwable.printStackTrace(); - - }); - - t1.start(); - JobTicket ticket = dmManager.startJob(queryBatcher); - - queryBatcher.awaitCompletion(); - t1.join(); - dmManager.stopJob(ticket); - - System.out.println("URI's size " + urisList.size()); - AtomicInteger successDocs = new AtomicInteger(); - Set uris2 = Collections.synchronizedSet(new HashSet<>()); - StringBuffer failures2 = new StringBuffer(); - - QueryBatcher deleteBatcher = dmManager.newQueryBatcher(urisList.iterator()) - .withBatchSize(13) - .withThreadCount(5) - .onUrisReady(new DeleteListener()) - .onUrisReady(batch -> successDocs.addAndGet(batch.getItems().length)) - .onUrisReady(batch -> uris2.addAll(Arrays.asList(batch.getItems()))) - .onQueryFailure(throwable -> { - throwable.printStackTrace(); - failures2.append("ERROR:[" + throwable + "]\n"); - }); - - JobTicket delTicket = dmManager.startJob(deleteBatcher); - deleteBatcher.awaitCompletion(); - dmManager.stopJob(delTicket); - - if (failures2.length() > 0) - fail(failures2.toString()); - - assertEquals(0, dbClient.newServerEval().xquery(query1).eval().next().getNumber().intValue()); - - DocumentPage page = dbClient.newDocumentManager().read("/local/json-1998"); - JacksonHandle dh = new JacksonHandle(); - while (page.hasNext()) { - DocumentRecord rec = page.next(); - rec.getContent(dh); - System.out.println("Results are: " + dh.get().get("k1").asText()); - - } - - props.put("merge-timestamp", "0"); - changeProperty(props, "/manage/v2/databases/" + dbName + "/properties"); +// if (true) return; + + setMergeTimestamp(DB_NAME, "-6000000000"); + + try { + // No idea what this is supposed to be doing, as the docs it's deleting don't ever exist. + // j initially was 1999 but that was causing the test to take about 20s, so lowered it. + class MyRunnable implements Runnable { + @Override + public void run() { + for (int j = 300; j >= 200; j--) { + dbClient.newDocumentManager().delete("/local/json-" + j); + } + } + } + Thread t1; + t1 = new Thread(new MyRunnable()); + + Set urisList = Collections.synchronizedSet(new HashSet<>()); + + QueryBatcher queryBatcher = dmManager.newQueryBatcher( + new StructuredQueryBuilder().collection("DeleteListener")) + .withBatchSize(11) + .withThreadCount(4) + .withConsistentSnapshot() + .onUrisReady(batch -> { + for (String s : batch.getItems()) { + urisList.add(s); + } + }) + .onQueryFailure(throwable -> { + System.out.println("Exceptions thrown from callback onQueryFailure"); + throwable.printStackTrace(); + + }); + + t1.start(); + JobTicket ticket = dmManager.startJob(queryBatcher); + + queryBatcher.awaitCompletion(); + t1.join(); + dmManager.stopJob(ticket); + + System.out.println("URI's size " + urisList.size()); + AtomicInteger successDocs = new AtomicInteger(); + Set uris2 = Collections.synchronizedSet(new HashSet<>()); + StringBuffer failures2 = new StringBuffer(); + + QueryBatcher deleteBatcher = dmManager.newQueryBatcher(urisList.iterator()) + .withBatchSize(13) + .withThreadCount(5) + .onUrisReady(new DeleteListener()) + .onUrisReady(batch -> successDocs.addAndGet(batch.getItems().length)) + .onUrisReady(batch -> uris2.addAll(Arrays.asList(batch.getItems()))) + .onQueryFailure(throwable -> { + throwable.printStackTrace(); + failures2.append("ERROR:[" + throwable + "]\n"); + }); + + JobTicket delTicket = dmManager.startJob(deleteBatcher); + deleteBatcher.awaitCompletion(); + dmManager.stopJob(delTicket); + + if (failures2.length() > 0) + fail(failures2.toString()); + + assertEquals(0, dbClient.newServerEval().xquery(query1).eval().next().getNumber().intValue()); + + DocumentPage page = dbClient.newDocumentManager().read("/local/json-1998"); + JacksonHandle dh = new JacksonHandle(); + while (page.hasNext()) { + DocumentRecord rec = page.next(); + rec.getContent(dh); + System.out.println("Results are: " + dh.get().get("k1").asText()); + + } + } finally { + setMergeTimestamp(DB_NAME, "0"); + } } @Test diff --git a/marklogic-client-api-functionaltests/src/test/java/com/marklogic/client/functionaltest/TestBug18736.java b/marklogic-client-api-functionaltests/src/test/java/com/marklogic/client/fastfunctest/TestBug18736.java similarity index 97% rename from marklogic-client-api-functionaltests/src/test/java/com/marklogic/client/functionaltest/TestBug18736.java rename to marklogic-client-api-functionaltests/src/test/java/com/marklogic/client/fastfunctest/TestBug18736.java index 97765a02c..90e078d66 100644 --- a/marklogic-client-api-functionaltests/src/test/java/com/marklogic/client/functionaltest/TestBug18736.java +++ b/marklogic-client-api-functionaltests/src/test/java/com/marklogic/client/fastfunctest/TestBug18736.java @@ -14,13 +14,14 @@ * limitations under the License. */ -package com.marklogic.client.functionaltest; +package com.marklogic.client.fastfunctest; import com.marklogic.client.DatabaseClient; import com.marklogic.client.admin.QueryOptionsManager; import com.marklogic.client.admin.ServerConfigurationManager; import com.marklogic.client.document.DocumentDescriptor; import com.marklogic.client.document.XMLDocumentManager; +import com.marklogic.client.fastfunctest.AbstractFunctionalTest; import com.marklogic.client.io.*; import com.marklogic.client.query.*; import com.marklogic.client.query.StructuredQueryBuilder.Operator; @@ -29,6 +30,7 @@ import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.skyscreamer.jsonassert.JSONAssert; import org.w3c.dom.Document; @@ -45,29 +47,15 @@ import static org.custommonkey.xmlunit.XMLAssert.assertXpathEvaluatesTo; import static org.junit.jupiter.api.Assertions.*; -public class TestBug18736 extends BasicJavaClientREST { - - private static String dbName = "Bug18736DB"; - private static String[] fNames = { "Bug18736DB-1" }; - - @BeforeAll - public static void setUp() throws Exception - { - System.out.println("In setup"); - configureRESTServer(dbName, fNames); - setupAppServicesConstraint(dbName); - } - - @AfterEach - public void afterEach() throws Exception - { - System.out.println("In afterEach"); - clearDB(); - } +public class TestBug18736 extends AbstractFunctionalTest { + @BeforeEach + void setup() { + deleteDocuments(newDatabaseClientBuilder().build()); + } @Test - public void testBug18736() throws KeyManagementException, NoSuchAlgorithmException, XpathException, TransformerException, ParserConfigurationException, SAXException, IOException + public void testBug18736() throws KeyManagementException, NoSuchAlgorithmException, TransformerException, IOException { System.out.println("Running testBug18736"); @@ -685,12 +673,4 @@ public void testBug22037() throws KeyManagementException, NoSuchAlgorithmExcepti // release client client.release(); } - - @AfterAll - public static void tearDown() throws Exception - { - System.out.println("In tear down"); - cleanupRESTServer(dbName, fNames); - - } } diff --git a/marklogic-client-api-functionaltests/src/test/java/com/marklogic/client/functionaltest/TestBug21159.java b/marklogic-client-api-functionaltests/src/test/java/com/marklogic/client/fastfunctest/TestBug21159.java similarity index 84% rename from marklogic-client-api-functionaltests/src/test/java/com/marklogic/client/functionaltest/TestBug21159.java rename to marklogic-client-api-functionaltests/src/test/java/com/marklogic/client/fastfunctest/TestBug21159.java index 61a33de90..9bb4109ad 100644 --- a/marklogic-client-api-functionaltests/src/test/java/com/marklogic/client/functionaltest/TestBug21159.java +++ b/marklogic-client-api-functionaltests/src/test/java/com/marklogic/client/fastfunctest/TestBug21159.java @@ -14,11 +14,12 @@ * limitations under the License. */ -package com.marklogic.client.functionaltest; +package com.marklogic.client.fastfunctest; import com.marklogic.client.DatabaseClient; import com.marklogic.client.admin.QueryOptionsManager; import com.marklogic.client.admin.ServerConfigurationManager; +import com.marklogic.client.fastfunctest.AbstractFunctionalTest; import com.marklogic.client.io.Format; import com.marklogic.client.io.StringHandle; import com.marklogic.client.io.TuplesHandle; @@ -41,33 +42,16 @@ import static org.junit.jupiter.api.Assertions.*; -public class TestBug21159 extends BasicJavaClientREST { - - private static String dbName = "TestTuplesRawCombinedQueryDB"; - private static String[] fNames = { "TestTuplesRawCombinedQueryDB-1" }; - - @BeforeAll - public static void setUp() throws Exception - { - System.out.println("In setup"); - configureRESTServer(dbName, fNames); - setupAppServicesConstraint(dbName); - - addRangeElementIndex(dbName, "string", "", "grandchild", "http://marklogic.com/collation/"); - addRangeElementIndex(dbName, "double", "", "double"); - addRangeElementIndex(dbName, "int", "", "int"); - addRangeElementIndex(dbName, "string", "", "string", "http://marklogic.com/collation/"); - } +public class TestBug21159 extends AbstractFunctionalTest { @Test - public void testBug21159Tuples() throws KeyManagementException, NoSuchAlgorithmException, IOException, ParserConfigurationException, SAXException, XpathException, - TransformerException + public void testBug21159Tuples() throws IOException { System.out.println("Running testBug21159Tuples"); String[] filenames = { "tuples-test1.xml", "tuples-test2.xml", "tuples-test3.xml", "tuples-test4.xml", "lexicon-test1.xml", "lexicon-test2.xml" }; - DatabaseClient client = getDatabaseClient("rest-admin", "x", getConnType()); + DatabaseClient client = newDatabaseClientBuilder().build(); // set query option validation to true ServerConfigurationManager srvMgr = client.newServerConfigManager(); @@ -110,14 +94,13 @@ public void testBug21159Tuples() throws KeyManagementException, NoSuchAlgorithmE } @Test - public void testBug21159Values() throws KeyManagementException, NoSuchAlgorithmException, IOException, ParserConfigurationException, SAXException, XpathException, - TransformerException + public void testBug21159Values() throws IOException { System.out.println("Running testBug21159Values"); String[] filenames = { "tuples-test1.xml", "tuples-test2.xml", "tuples-test3.xml", "tuples-test4.xml", "lexicon-test1.xml", "lexicon-test2.xml" }; - DatabaseClient client = getDatabaseClient("rest-admin", "x", getConnType()); + DatabaseClient client = newDatabaseClientBuilder().build(); // set query option validation to true ServerConfigurationManager srvMgr = client.newServerConfigManager(); @@ -157,15 +140,14 @@ public void testBug21159Values() throws KeyManagementException, NoSuchAlgorithmE } @Test - public void testTuplesWithRawCtsQueryDefinition() throws KeyManagementException, NoSuchAlgorithmException, IOException, ParserConfigurationException, SAXException, XpathException, - TransformerException + public void testTuplesWithRawCtsQueryDefinition() throws IOException { System.out.println("Running testTuplesWithRawCtsQueryDefinition"); String queryOptionName = "tuplesTest"; String[] filenames = { "tuples-test1.xml", "tuples-test2.xml", "tuples-test3.xml", "tuples-test4.xml", "lexicon-test1.xml", "lexicon-test2.xml" }; - DatabaseClient client = getDatabaseClient("rest-admin", "x", getConnType()); + DatabaseClient client = newDatabaseClientBuilder().build(); // set query option validation to true ServerConfigurationManager srvMgr = client.newServerConfigManager(); srvMgr.readConfiguration(); @@ -236,13 +218,12 @@ public void testTuplesWithRawCtsQueryDefinition() throws KeyManagementException, } @Test - public void testTuplesWithRawCombinedCtsQuery() throws KeyManagementException, NoSuchAlgorithmException, IOException, ParserConfigurationException, SAXException, XpathException, - TransformerException + public void testTuplesWithRawCombinedCtsQuery() throws IOException { System.out.println("Running testTuplesWithRawCombinedCtsQuery"); String[] filenames = { "tuples-test1.xml", "tuples-test2.xml", "tuples-test3.xml", "tuples-test4.xml", "lexicon-test1.xml", "lexicon-test2.xml" }; - DatabaseClient client = getDatabaseClient("rest-admin", "x", getConnType()); + DatabaseClient client = newDatabaseClientBuilder().build(); // set query option validation to true ServerConfigurationManager srvMgr = client.newServerConfigManager(); srvMgr.readConfiguration(); @@ -311,11 +292,4 @@ public void testTuplesWithRawCombinedCtsQuery() throws KeyManagementException, N // release client client.release(); } - - @AfterAll - public static void tearDown() throws Exception - { - System.out.println("In tear down"); - cleanupRESTServer(dbName, fNames); - } } diff --git a/marklogic-client-api-functionaltests/src/test/java/com/marklogic/client/functionaltest/TestDatabaseClientConnection.java b/marklogic-client-api-functionaltests/src/test/java/com/marklogic/client/fastfunctest/TestDatabaseClientConnection.java similarity index 82% rename from marklogic-client-api-functionaltests/src/test/java/com/marklogic/client/functionaltest/TestDatabaseClientConnection.java rename to marklogic-client-api-functionaltests/src/test/java/com/marklogic/client/fastfunctest/TestDatabaseClientConnection.java index 0c7c4f5c9..684d4d27f 100644 --- a/marklogic-client-api-functionaltests/src/test/java/com/marklogic/client/functionaltest/TestDatabaseClientConnection.java +++ b/marklogic-client-api-functionaltests/src/test/java/com/marklogic/client/fastfunctest/TestDatabaseClientConnection.java @@ -14,7 +14,7 @@ * limitations under the License. */ -package com.marklogic.client.functionaltest; +package com.marklogic.client.fastfunctest; import com.marklogic.client.DatabaseClient; import com.marklogic.client.DatabaseClientFactory; @@ -25,14 +25,27 @@ import com.marklogic.client.alerting.RuleDefinition; import com.marklogic.client.alerting.RuleDefinitionList; import com.marklogic.client.alerting.RuleManager; -import com.marklogic.client.document.*; +import com.marklogic.client.document.DocumentDescriptor; import com.marklogic.client.document.DocumentManager.Metadata; -import com.marklogic.client.io.*; +import com.marklogic.client.document.DocumentPage; +import com.marklogic.client.document.DocumentUriTemplate; +import com.marklogic.client.document.DocumentWriteSet; +import com.marklogic.client.document.TextDocumentManager; +import com.marklogic.client.fastfunctest.AbstractFunctionalTest; +import com.marklogic.client.io.DocumentMetadataHandle; import com.marklogic.client.io.DocumentMetadataHandle.DocumentMetadataValues; -import com.marklogic.client.query.*; +import com.marklogic.client.io.InputStreamHandle; +import com.marklogic.client.io.StringHandle; +import com.marklogic.client.io.TuplesHandle; +import com.marklogic.client.io.ValuesHandle; +import com.marklogic.client.io.ValuesListHandle; +import com.marklogic.client.query.AggregateResult; +import com.marklogic.client.query.QueryManager; +import com.marklogic.client.query.StringQueryDefinition; +import com.marklogic.client.query.SuggestDefinition; +import com.marklogic.client.query.ValuesDefinition; +import com.marklogic.client.query.ValuesListDefinition; import org.custommonkey.xmlunit.exceptions.XpathException; -import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; import org.w3c.dom.Document; import org.xml.sax.SAXException; @@ -41,7 +54,13 @@ import javax.net.ssl.X509TrustManager; import javax.xml.parsers.ParserConfigurationException; import javax.xml.transform.TransformerException; -import java.io.*; +import java.io.BufferedReader; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.FileReader; +import java.io.IOException; +import java.io.InputStream; import java.security.KeyManagementException; import java.security.KeyStoreException; import java.security.NoSuchAlgorithmException; @@ -55,46 +74,11 @@ import java.util.TreeMap; import static org.custommonkey.xmlunit.XMLAssert.assertXMLEqual; -import static org.junit.jupiter.api.Assertions.*; - -public class TestDatabaseClientConnection extends BasicJavaClientREST { - - private static String dbName = "DatabaseClientConnectionDB"; - private static String[] fNames = { "DatabaseClientConnectionDB-1" }; - private static String restServerName; - - // These members are used to test Git Issue 332. - private static String UberdbName = "UberDatabaseClientConnectionDB"; - private static String[] UberfNames = { "UberDatabaseClientConnectionDB-1" }; - private static int Uberport = 8000; - private static String UberrestServerName = "App-Services"; - private static String appServerHostname = null; - - @BeforeAll - public static void setUp() throws Exception { - System.out.println("In setup"); - configureRESTServer(dbName, fNames); - restServerName = getRestServerName(); - appServerHostname = getRestAppServerHostName(); - - /* - * Only users with the http://marklogic.com/xdmp/privileges/xdmp-eval-in - * (xdmp:eval-in) or equivalent privilege can send request parameter that - * enables the request to be evaluated against a content database other than - * the default database associated with the REST API instances - */ - createUserRolesWithPrevilages("test-eval", "xdbc:eval", "xdbc:eval-in", "xdmp:eval-in", "any-uri", "xdbc:invoke"); - createRESTUser("eval-user", "x", "test-eval", "rest-admin", "rest-writer", "rest-reader", "rest-extension-user", "manage-user"); - - // Create a database and forest for use on uber port (8000). Do not - // associate the REST server (on 8000) with an DB. - createDB(UberdbName); - createForest(UberfNames[0], UberdbName); - - setupAppServicesConstraint(UberdbName); - addRangeElementIndex(UberdbName, "string", "http://action/", "title", "http://marklogic.com/collation/"); - addRangeElementIndex(UberdbName, "string", "http://noun/", "title", "http://marklogic.com/collation/"); - } +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class TestDatabaseClientConnection extends AbstractFunctionalTest { @Test public void testReleasedClient() throws IOException, KeyManagementException, NoSuchAlgorithmException @@ -169,8 +153,7 @@ public X509Certificate[] getAcceptedIssuers() { }) .withSSLHostnameVerifier(SSLHostnameVerifier.ANY); - client = DatabaseClientFactory.newClient(getRestServerHostName(), getRestServerPort(), - secContext, getConnType()); + client = newDatabaseClientBuilder().withSecurityContext(secContext).build(); SecurityContext readSecContext = client.getSecurityContext(); String verifier = readSecContext.getSSLHostnameVerifier().toString(); String protocol = readSecContext.getSSLContext().getProtocol(); @@ -191,8 +174,7 @@ public void testDatabaseClientConnectionInvalidPort() throws IOException String filename = "facebook-10443244874876159931"; - SecurityContext secContext = newSecurityContext("rest-reader", "x"); - DatabaseClient client = DatabaseClientFactory.newClient(appServerHostname, 8033, secContext, getConnType()); + DatabaseClient client = newDatabaseClientBuilder().withPort(8033).build(); String expectedException = null; String exception = ""; @@ -272,14 +254,13 @@ public void testDatabaseClientConnectionInvalidPassword() throws IOException, Ke } @Test - public void testDatabaseClientConnectionInvalidHost() throws IOException + public void testDatabaseClientConnectionInvalidHost() { System.out.println("Running testDatabaseClientConnectionInvalidHost"); String filename = "facebook-10443244874876159931"; - SecurityContext secContext = newSecurityContext("rest-writer", "x"); - DatabaseClient client = DatabaseClientFactory.newClient("foobarhost", 8011, secContext, getConnType()); + DatabaseClient client = newDatabaseClientBuilder().withHost("badhost").build(); // String expectedException = // "com.sun.jersey.api.client.ClientHandlerException: java.net.UnknownHostException: foobarhost: Name or service not known"; @@ -305,17 +286,6 @@ public void testDatabaseClientConnectionInvalidHost() throws IOException /* * These tests are specifically to validate Git Issue 332. * https://github.com/marklogic/java-client-api/issues/332 - * - * We need to test that REST calls can pass a database name and access that - * database on the uber server port (8000). Create a database, forest and - * associate the database to the uber server on port 8000, where App-Services - * is running. We will be testing the following : - * - * QueryManager.suggest() QueryManager.tuples() QueryManager.values() - * QueryManager.valuesList() Transaction.readStatus() - * RuleManager.readRule(As)() RuleManager.match(As)() with - * StructureWriteHandle with StringQueryDefinition with - * StructuredQueryDefinition with RawQueryDefinition with docIds */ // Trying to access database without specifying the database name. @@ -326,8 +296,7 @@ public void testDBClientUsingWithoutDatabaseName() throws IOException, SAXExcept String filename = "xml-original-test.xml"; String uri = "/write-xml-string/"; - SecurityContext secContext = newSecurityContext("eval-user", "x"); - DatabaseClient client = DatabaseClientFactory.newClient(appServerHostname, Uberport, secContext, getConnType()); + DatabaseClient client = newDatabaseClientBuilder().build(); // write doc writeDocumentUsingStringHandle(client, filename, uri, "XML"); @@ -349,19 +318,13 @@ public void testDBClientUsingWithoutDatabaseName() throws IOException, SAXExcept // Trying to access database by specifying the database name. @Test - public void testDBClientUsingWithDatabaseName() throws IOException, SAXException, ParserConfigurationException, KeyManagementException, NoSuchAlgorithmException + public void testDBClientUsingWithDatabaseName() throws IOException, SAXException, ParserConfigurationException { System.out.println("Running testDBClientUsingWithDatabaseName"); String filename = "xml-original-test.xml"; String uri = "/write-xml-string/"; - DatabaseClient client = null; - if (isLBHost()) - client = getDatabaseClient("eval-user", "x", getConnType()); - else { - SecurityContext secContext = newSecurityContext("eval-user", "x"); - client = DatabaseClientFactory.newClient(appServerHostname, Uberport, UberdbName, secContext, getConnType()); - } + DatabaseClient client = newDatabaseClientBuilder().withDatabase(DB_NAME).build(); // write doc writeDocumentUsingStringHandle(client, filename, uri, "XML"); @@ -389,8 +352,7 @@ public void testUberSearchSuggestionMultiByte() throws FileNotFoundException String[] filenames = { "multibyte1.xml", "multibyte2.xml", "multibyte3.xml" }; String queryOptionName = "suggestionOpt.xml"; - SecurityContext secContext = newSecurityContext("eval-user", "x"); - DatabaseClient client = DatabaseClientFactory.newClient(appServerHostname, Uberport, UberdbName, secContext, getConnType()); + DatabaseClient client = newDatabaseClientBuilder().build(); // write docs for (String filename : filenames) { writeDocumentUsingInputStreamHandle(client, filename, "/ss/", "XML"); @@ -412,15 +374,14 @@ public void testUberSearchSuggestionMultiByte() throws FileNotFoundException } @Test - public void testQueryManagerTuples() throws IOException, ParserConfigurationException, SAXException, XpathException, TransformerException + public void testQueryManagerTuples() throws IOException { System.out.println("Running testQueryManagerTuples"); - String[] filenames = { "aggr1.xml", "aggr2.xml", "aggr3.xml", "aggr4.xml", "aggr5.xml" }; String queryOptionName = "aggregatesOpt.xml"; - SecurityContext secContext = newSecurityContext("eval-user", "x"); - DatabaseClient client = DatabaseClientFactory.newClient(appServerHostname, Uberport, UberdbName, secContext, getConnType()); + DatabaseClient client = newDatabaseClientBuilder().build(); + deleteDocuments(client); // write docs for (String filename : filenames) { @@ -453,8 +414,8 @@ public void testQueryManagerTuples() throws IOException, ParserConfigurationExce System.out.println(roundedCorrelation); System.out.println(roundedCovariance); - assertEquals( "0.37", roundedCorrelation); - assertEquals( "0.48", roundedCovariance); + assertEquals( "0.26", roundedCorrelation); + assertEquals( "0.35", roundedCovariance); ValuesListDefinition vdef = queryMgr.newValuesListDefinition("aggregatesOpt.xml"); ValuesListHandle results = queryMgr.valuesList(vdef, new ValuesListHandle()); @@ -470,15 +431,14 @@ public void testQueryManagerTuples() throws IOException, ParserConfigurationExce } @Test - public void testValuesOccurences() throws IOException, ParserConfigurationException, SAXException, XpathException, TransformerException + public void testValuesOccurrences() throws IOException { System.out.println("Running testValuesOccurences"); String[] filenames = { "aggr1.xml", "aggr2.xml", "aggr3.xml", "aggr4.xml" }; String queryOptionName = "aggregatesOpt5Occ.xml"; - SecurityContext secContext = newSecurityContext("eval-user", "x"); - DatabaseClient client = DatabaseClientFactory.newClient(appServerHostname, Uberport, UberdbName, secContext, getConnType()); + DatabaseClient client = newDatabaseClientBuilder().withDatabase(DB_NAME).build(); // write docs for (String filename : filenames) { @@ -507,13 +467,12 @@ public void testValuesOccurences() throws IOException, ParserConfigurationExcept } @Test - public void testTransactionReadStatus() throws Exception { + public void testTransactionReadStatus() { System.out.println("Running testTransactionReadStatus"); String docId[] = { "/foo/test/transactionURIFoo1.txt", "/foo/test/transactionURIFoo2.txt", "/foo/test/transactionURIFoo3.txt" }; - SecurityContext secContext = newSecurityContext("eval-user", "x"); - DatabaseClient client = DatabaseClientFactory.newClient(appServerHostname, Uberport, UberdbName, secContext, getConnType()); + DatabaseClient client = newDatabaseClientBuilder().build(); Transaction transaction = client.openTransaction(); try { TextDocumentManager docMgr = client.newTextDocumentManager(); @@ -526,8 +485,7 @@ public void testTransactionReadStatus() throws Exception { docMgr.write(writeset, transaction); StringHandle wrteTransHandle = new StringHandle(); transaction.readStatus(wrteTransHandle); - assertTrue((wrteTransHandle.get()).contains(UberrestServerName)); - assertTrue((wrteTransHandle.get()).contains("App-Services")); + assertTrue((wrteTransHandle.get()).contains(getRestServerName())); transaction.commit(); transaction = client.openTransaction(); @@ -536,8 +494,7 @@ public void testTransactionReadStatus() throws Exception { assertTrue(page.size() == 3); StringHandle readTransHandle = new StringHandle(); transaction.readStatus(readTransHandle); - assertTrue((readTransHandle.get()).contains(UberrestServerName)); - assertTrue((readTransHandle.get()).contains("App-Services")); + assertTrue((readTransHandle.get()).contains(getRestServerName())); } catch (Exception exp) { System.out.println(exp.getMessage()); @@ -548,8 +505,7 @@ public void testTransactionReadStatus() throws Exception { } @Test - public void testRMMatchQDAndDocIds() throws IOException, ParserConfigurationException, SAXException, XpathException, TransformerException, KeyManagementException, - NoSuchAlgorithmException + public void testRMMatchQDAndDocIds() throws IOException, KeyManagementException, NoSuchAlgorithmException { System.out.println("Running testRMMatchQDAndDocIds"); String[] filenames = { "constraint1.xml", "constraint2.xml", "constraint3.xml", "constraint4.xml", "constraint5.xml" }; @@ -631,8 +587,7 @@ public void testAddAs() throws Exception { System.out.println("Running testAddAs"); String[] docId = { "aggr1.xml", "aggr2.xml", "aggr3.xml" }; - SecurityContext secContext = newSecurityContext("eval-user", "x"); - DatabaseClient client = DatabaseClientFactory.newClient(appServerHostname, Uberport, UberdbName, secContext, getConnType()); + DatabaseClient client = newDatabaseClientBuilder().build(); Transaction transaction = client.openTransaction(); try { @@ -650,8 +605,8 @@ public void testAddAs() throws Exception { docMgr.write(writeset, transaction); StringHandle wrteTransHandle = new StringHandle(); transaction.readStatus(wrteTransHandle); - assertTrue((wrteTransHandle.get()).contains(UberrestServerName)); - assertTrue((wrteTransHandle.get()).contains("App-Services")); + System.out.println(wrteTransHandle.get()); + assertTrue((wrteTransHandle.get()).contains(getRestServerName())); transaction.commit(); transaction = client.openTransaction(); @@ -669,8 +624,8 @@ public void testAddAs() throws Exception { StringHandle readTransHandle = new StringHandle(); transaction.readStatus(readTransHandle); - assertTrue((readTransHandle.get()).contains(UberrestServerName)); - assertTrue((readTransHandle.get()).contains(dbName)); + assertTrue((readTransHandle.get()).contains(getRestServerName())); + assertTrue((readTransHandle.get()).contains(DB_NAME)); assertTrue((readTransHandle.get()).contains(txId)); } catch (Exception exp) { System.out.println(exp.getMessage()); @@ -692,8 +647,7 @@ public void testRuleManagerReadAs() throws IOException, ParserConfigurationExcep try { String[] filenames = { "constraint1.xml", "constraint2.xml", "constraint3.xml", "constraint4.xml", "constraint5.xml" }; - SecurityContext secContext = newSecurityContext("eval-user", "x"); - client = DatabaseClientFactory.newClient(appServerHostname, Uberport, UberdbName, secContext, getConnType()); + client = newDatabaseClientBuilder().build(); // write docs for (String filename : filenames) { writeDocumentUsingInputStreamHandle(client, filename, "/raw-alert/", "XML"); @@ -1012,22 +966,4 @@ public void testRuleManagerMatchAs() throws IOException, ParserConfigurationExce // release client client.release(); } - - @AfterAll - public static void tearDown() throws Exception { - System.out.println("In tear down"); - - if (!IsSecurityEnabled()) { - setAuthenticationAndDefaultUser(restServerName, authType, "nobody"); - } - // Associate the Server with Documents. Due to test orders being - // undeterministic not sure which DB will be associated. - associateRESTServerWithDB(restServerName, "Documents"); - cleanupRESTServer(dbName, fNames); - deleteDB(UberdbName); - deleteForest(UberfNames[0]); - - deleteRESTUser("eval-user"); - deleteUserRole("test-eval"); - } } diff --git a/marklogic-client-api-functionaltests/src/test/java/com/marklogic/client/fastfunctest/TestPointInTimeQuery.java b/marklogic-client-api-functionaltests/src/test/java/com/marklogic/client/fastfunctest/TestPointInTimeQuery.java new file mode 100644 index 000000000..f7f6d4b8b --- /dev/null +++ b/marklogic-client-api-functionaltests/src/test/java/com/marklogic/client/fastfunctest/TestPointInTimeQuery.java @@ -0,0 +1,210 @@ +/* + * Copyright (c) 2023 MarkLogic Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.marklogic.client.fastfunctest; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.marklogic.client.DatabaseClient; +import com.marklogic.client.document.DocumentManager; +import com.marklogic.client.document.DocumentPatchBuilder; +import com.marklogic.client.document.DocumentPatchBuilder.PathLanguage; +import com.marklogic.client.document.DocumentPatchBuilder.Position; +import com.marklogic.client.document.JSONDocumentManager; +import com.marklogic.client.fastfunctest.AbstractFunctionalTest; +import com.marklogic.client.io.JacksonHandle; +import com.marklogic.client.io.StringHandle; +import com.marklogic.client.io.marker.DocumentPatchHandle; +import com.marklogic.rest.util.Fragment; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.security.KeyManagementException; +import java.security.NoSuchAlgorithmException; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class TestPointInTimeQuery extends AbstractFunctionalTest { + + @BeforeAll + public static void setUp() throws Exception { + setMergeTimestamp(DB_NAME, "-600000000"); + } + + @AfterAll + public static void afterAll() { + setMergeTimestamp(DB_NAME, "0"); + } + + /* + * This test verifies if fragments are available for a document that is + * inserted and then updated when merge time is set to 60 seconds and both + * insert and update are within that time period. + * + * Git Issue 457 needs to be completed in order for this test to be fleshed + * completly. + * + * Insert doc Verify fragment counts Update the document Verify read with + * Point In Time Stamp Verify fragment counts. Update again. Verify read with + * Point In Time Stamp again Verify fragment counts second time + */ + @Test + public void testAInsertAndUpdateJson() throws KeyManagementException, NoSuchAlgorithmException, IOException { + String[] filenames = {"json-original.json"}; + + DatabaseClient client = getDatabaseClient("rest-evaluator", "x", getConnType()); + + DatabaseCounts originalCounts = getFragmentCounts(); + + // write docs and save the timestamps in the array + + for (String filename : filenames) { + writeDocumentUsingInputStreamHandle(client, filename, "/partial-update/", "JSON"); + } + DocumentManager docMgrIns = client.newJSONDocumentManager(); + // create handle + JacksonHandle jacksonHandle = new JacksonHandle(); + + // Read the document with timestamp. + docMgrIns.read("/partial-update/" + filenames[0], jacksonHandle); + + // Make sure we have the original document and document count is 1 + + long insTimeStamp = jacksonHandle.getServerTimestamp(); + System.out.println("Point in Time Stamp after the initial insert " + insTimeStamp); + DatabaseCounts dbCounts = getFragmentCounts(); + System.out.println("Fragment counts after initial insert: " + dbCounts); + assertEquals(originalCounts.activeFragments + 1, dbCounts.activeFragments); + assertEquals(originalCounts.deletedFragments, dbCounts.deletedFragments); + + // Update the doc. Insert a fragment. + ObjectMapper mapper = new ObjectMapper(); + + String docId = "/partial-update/json-original.json"; + JSONDocumentManager docMgr = client.newJSONDocumentManager(); + DocumentPatchBuilder patchBldr = docMgr.newPatchBuilder(); + patchBldr.pathLanguage(PathLanguage.JSONPATH); + + ObjectNode fragmentNode = mapper.createObjectNode(); + ObjectNode fragmentNode1 = mapper.createObjectNode(); + ObjectNode fragmentNode2 = mapper.createObjectNode(); + + fragmentNode.put("insertedKey", 9); + fragmentNode1.put("original", true); + fragmentNode2.put("modified", false); + + String fragment = mapper.writeValueAsString(fragmentNode); + String fragment1 = mapper.writeValueAsString(fragmentNode1); + String fragment2 = mapper.writeValueAsString(fragmentNode2); + + String jsonpath = new String("$.employees[2]"); + patchBldr.insertFragment(jsonpath, Position.AFTER, fragment); + patchBldr.insertFragment("$.employees[2]", Position.AFTER, fragment1); + + DocumentPatchHandle patchHandle = patchBldr.build(); + docMgr.patch(docId, patchHandle); + + // Now read using a handle and time value in insTimeStamp. The updates + // should not be present. + StringHandle shReadInsTS = new StringHandle(); + shReadInsTS.setPointInTimeQueryTimestamp(insTimeStamp); + String insTS = docMgr.read(docId, shReadInsTS).get(); + System.out.println(insTS); + assertFalse(insTS.contains("{\"insertedKey\":9}")); + assertFalse(insTS.contains("{\"original\":true}")); + + // Now read using a handle without a Point In Time value set on the handle. + // Should return the latest document. + String content = docMgr.read(docId, new StringHandle()).get(); + + System.out.println(content); + + assertTrue(content.contains("{\"insertedKey\":9}")); + assertTrue(content.contains("{\"original\":true}")); + + // Read the document with timestamp. Todo QUERY DB for document at a Point + // In Time. Git Issue 457 + // Verify that the test can validate the first original docs only within the + // -60 (time interval) + + // Sleep for some time. Read again with new point In time. + // Verify that we get updated document. + + docMgrIns.read("/partial-update/" + filenames[0], jacksonHandle); + + long firstUpdTimeStamp = jacksonHandle.getPointInTimeQueryTimestamp(); + System.out.println("Point in Time Stamp after the first update " + firstUpdTimeStamp); + + // Verify the counts + dbCounts = getFragmentCounts(); + System.out.println("Fragment counts after first update: " + dbCounts); + assertEquals(originalCounts.activeFragments + 1, dbCounts.activeFragments); + assertEquals(originalCounts.deletedFragments + 1, dbCounts.deletedFragments); + + // Insert / update the document again + + patchBldr.insertFragment("$.employees[0]", Position.AFTER, fragment2); + + patchHandle = patchBldr.build(); + docMgr.patch(docId, patchHandle); + + content = docMgr.read(docId, new StringHandle()).get(); + System.out.println(content); + assertTrue(content.contains("{\"modified\":false}")); + + // Read the document with timestamp. Todo QUERY DB for document at a Point + // In Time. Git Issue 457 + // Verify that the test can validate the first original docs only within the + // -60 (time interval) + + // Sleep for some time. Read again with new point In time. + // Verify that we get updated document. + + docMgrIns.read("/partial-update/" + filenames[0], jacksonHandle); + + long secondUpdTimeStamp = jacksonHandle.getPointInTimeQueryTimestamp(); + System.out.println("Point in Time Stamp after the first update " + secondUpdTimeStamp); + + dbCounts = getFragmentCounts(); + System.out.println("Fragment counts after second update: " + dbCounts); + assertEquals(originalCounts.activeFragments + 1, dbCounts.activeFragments); + assertEquals(originalCounts.deletedFragments + 2, dbCounts.deletedFragments); + + client.release(); + } + + static class DatabaseCounts { + public int activeFragments; + public int deletedFragments; + + @Override + public String toString() { + return "active=" + activeFragments + "; deleted=" + deletedFragments; + } + } + + private DatabaseCounts getFragmentCounts() { + Fragment doc = newManageClient().getXml("/manage/v2/databases/" + DB_NAME + "?view=counts&format=xml"); + DatabaseCounts dbCounts = new DatabaseCounts(); + dbCounts.activeFragments = Integer.parseInt(doc.getElementValue("/db:database-counts/db:count-properties/db:active-fragments")); + dbCounts.deletedFragments = Integer.parseInt(doc.getElementValue("/db:database-counts/db:count-properties/db:deleted-fragments")); + return dbCounts; + } +} diff --git a/marklogic-client-api-functionaltests/src/test/java/com/marklogic/client/functionaltest/ConnectedRESTQA.java b/marklogic-client-api-functionaltests/src/test/java/com/marklogic/client/functionaltest/ConnectedRESTQA.java index c540e54cb..22b980c0a 100644 --- a/marklogic-client-api-functionaltests/src/test/java/com/marklogic/client/functionaltest/ConnectedRESTQA.java +++ b/marklogic-client-api-functionaltests/src/test/java/com/marklogic/client/functionaltest/ConnectedRESTQA.java @@ -989,6 +989,10 @@ public static void tearDownJavaRESTServer(String dbName, String[] fNames, String logTestMessages(" Ending TESTCASE TEARDOWN ", beforeTeardown); } + public static void setMergeTimestamp(String dbName, String value) { + setDatabaseProperties(dbName, "merge-timestamp", value); + } + public static void setDatabaseProperties(String dbName, String prop, String propValue) { ObjectNode properties = new ObjectMapper().createObjectNode(); properties.put("database-name", dbName); diff --git a/marklogic-client-api-functionaltests/src/test/java/com/marklogic/client/functionaltest/TestPointInTimeQuery.java b/marklogic-client-api-functionaltests/src/test/java/com/marklogic/client/functionaltest/TestPointInTimeQuery.java deleted file mode 100644 index e4309aefd..000000000 --- a/marklogic-client-api-functionaltests/src/test/java/com/marklogic/client/functionaltest/TestPointInTimeQuery.java +++ /dev/null @@ -1,387 +0,0 @@ -/* - * Copyright (c) 2023 MarkLogic Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.marklogic.client.functionaltest; - -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.node.ObjectNode; -import com.marklogic.client.DatabaseClient; -import com.marklogic.client.document.DocumentManager; -import com.marklogic.client.document.DocumentPatchBuilder; -import com.marklogic.client.document.DocumentPatchBuilder.PathLanguage; -import com.marklogic.client.document.DocumentPatchBuilder.Position; -import com.marklogic.client.document.JSONDocumentManager; -import com.marklogic.client.io.JacksonHandle; -import com.marklogic.client.io.StringHandle; -import com.marklogic.client.io.marker.DocumentPatchHandle; -import org.apache.http.HttpResponse; -import org.apache.http.auth.AuthScope; -import org.apache.http.auth.UsernamePasswordCredentials; -import org.apache.http.client.methods.HttpGet; -import org.apache.http.impl.client.DefaultHttpClient; -import org.custommonkey.xmlunit.exceptions.XpathException; -import org.jdom2.Document; -import org.jdom2.Element; -import org.jdom2.Namespace; -import org.jdom2.input.SAXBuilder; -import org.junit.jupiter.api.*; -import org.xml.sax.SAXException; - -import javax.xml.parsers.ParserConfigurationException; -import javax.xml.transform.TransformerException; -import java.io.IOException; -import java.security.KeyManagementException; -import java.security.NoSuchAlgorithmException; - -import static org.junit.jupiter.api.Assertions.*; - -@TestMethodOrder(MethodOrderer.MethodName.class) -public class TestPointInTimeQuery extends BasicJavaClientREST { - private static String dbName = "TestPointInTimeQueryDB"; - private static String[] fNames = { "TestPointInTimeQuery-1" }; - private static int adminPort = 0; - private static String appServerHostname = null; - - @BeforeAll - public static void setUp() throws Exception { - System.out.println("In setup"); - configureRESTServer(dbName, fNames); - setupAppServicesConstraint(dbName); - - createUserRolesWithPrevilages("test-eval", "xdbc:eval", "xdbc:eval-in", "xdmp:eval-in", "any-uri", "xdbc:invoke"); - createRESTUser("eval-user", "x", "test-eval", "rest-admin", "rest-writer", "rest-reader"); - // set to - 60 seconds - setDatabaseProperties(dbName, "merge-timestamp", "-600000000"); - appServerHostname = getRestAppServerHostName(); - adminPort = getAdminPort(); - } - - @AfterEach - public void testCleanUp() throws Exception { - clearDB(); - System.out.println("Running clear script"); - } - - /* - * This test verifies if fragments are available for a document that is - * inserted and then updated when merge time is set to 60 seconds and both - * insert and update are within that time period. - * - * Git Issue 457 needs to be completed in order for this test to be fleshed - * completly. - * - * Insert doc Verify fragment counts Update the document Verify read with - * Point In Time Stamp Verify fragment counts. Update again. Verify read with - * Point In Time Stamp again Verify fragment counts second time - */ - @Test - public void testAInsertAndUpdateJson() throws KeyManagementException, NoSuchAlgorithmException, IOException - { - System.out.println("Running testAInsertAndUpdateJson"); - - String[] filenames = { "json-original.json" }; - - DatabaseClient client = getDatabaseClient("eval-user", "x", getConnType()); - - // write docs and save the timestamps in the array - - for (String filename : filenames) { - writeDocumentUsingInputStreamHandle(client, filename, "/partial-update/", "JSON"); - } - DocumentManager docMgrIns = client.newJSONDocumentManager(); - // create handle - JacksonHandle jacksonHandle = new JacksonHandle(); - - // Read the document with timestamp. - docMgrIns.read("/partial-update/" + filenames[0], jacksonHandle); - - // Make sure we have the original document and document count is 1 - - long insTimeStamp = jacksonHandle.getServerTimestamp(); - System.out.println("Point in Time Stamp after the initial insert " + insTimeStamp); - DatabaseCounts dbCounts = getDBFragmentCount(); - System.out.println("Fragment counts after initial insert: " + dbCounts); - assertEquals(1, dbCounts.activeFragments); - assertEquals(0, dbCounts.deletedFragments); - - // Update the doc. Insert a fragment. - ObjectMapper mapper = new ObjectMapper(); - - String docId = "/partial-update/json-original.json"; - JSONDocumentManager docMgr = client.newJSONDocumentManager(); - DocumentPatchBuilder patchBldr = docMgr.newPatchBuilder(); - patchBldr.pathLanguage(PathLanguage.JSONPATH); - - ObjectNode fragmentNode = mapper.createObjectNode(); - ObjectNode fragmentNode1 = mapper.createObjectNode(); - ObjectNode fragmentNode2 = mapper.createObjectNode(); - - fragmentNode.put("insertedKey", 9); - fragmentNode1.put("original", true); - fragmentNode2.put("modified", false); - - String fragment = mapper.writeValueAsString(fragmentNode); - String fragment1 = mapper.writeValueAsString(fragmentNode1); - String fragment2 = mapper.writeValueAsString(fragmentNode2); - - String jsonpath = new String("$.employees[2]"); - patchBldr.insertFragment(jsonpath, Position.AFTER, fragment); - patchBldr.insertFragment("$.employees[2]", Position.AFTER, fragment1); - - DocumentPatchHandle patchHandle = patchBldr.build(); - docMgr.patch(docId, patchHandle); - - // Now read using a handle and time value in insTimeStamp. The updates - // should not be present. - StringHandle shReadInsTS = new StringHandle(); - shReadInsTS.setPointInTimeQueryTimestamp(insTimeStamp); - String insTS = docMgr.read(docId, shReadInsTS).get(); - System.out.println(insTS); - assertFalse(insTS.contains("{\"insertedKey\":9}")); - assertFalse(insTS.contains("{\"original\":true}")); - - // Now read using a handle without a Point In Time value set on the handle. - // Should return the latest document. - String content = docMgr.read(docId, new StringHandle()).get(); - - System.out.println(content); - - assertTrue(content.contains("{\"insertedKey\":9}")); - assertTrue(content.contains("{\"original\":true}")); - - // Read the document with timestamp. Todo QUERY DB for document at a Point - // In Time. Git Issue 457 - // Verify that the test can validate the first original docs only within the - // -60 (time interval) - - // Sleep for some time. Read again with new point In time. - // Verify that we get updated document. - - docMgrIns.read("/partial-update/" + filenames[0], jacksonHandle); - - long firstUpdTimeStamp = jacksonHandle.getPointInTimeQueryTimestamp(); - System.out.println("Point in Time Stamp after the first update " + firstUpdTimeStamp); - - // Verify the counts - dbCounts = getDBFragmentCount(); - System.out.println("Fragment counts after first update: " + dbCounts); - assertEquals(1, dbCounts.activeFragments); - assertEquals(1, dbCounts.deletedFragments); - - // Insert / update the document again - - patchBldr.insertFragment("$.employees[0]", Position.AFTER, fragment2); - - patchHandle = patchBldr.build(); - docMgr.patch(docId, patchHandle); - - content = docMgr.read(docId, new StringHandle()).get(); - System.out.println(content); - assertTrue(content.contains("{\"modified\":false}")); - - // Read the document with timestamp. Todo QUERY DB for document at a Point - // In Time. Git Issue 457 - // Verify that the test can validate the first original docs only within the - // -60 (time interval) - - // Sleep for some time. Read again with new point In time. - // Verify that we get updated document. - - docMgrIns.read("/partial-update/" + filenames[0], jacksonHandle); - - long secondUpdTimeStamp = jacksonHandle.getPointInTimeQueryTimestamp(); - System.out.println("Point in Time Stamp after the first update " + secondUpdTimeStamp); - - dbCounts = getDBFragmentCount(); - System.out.println("Fragment counts after second update: " + dbCounts); - assertEquals(1, dbCounts.activeFragments); - assertEquals(2, dbCounts.deletedFragments); - - client.release(); - } - - /* - * This test verifies if fragments are available for a document that is - * inserted and then deleted when merge time is set to 60 seconds and both - * insert and delete are within that time period. - * - * Git Issue 457 needs to be completed in order for this test to be fleshed - * completly. - * - * Insert doc Read the doc and Verify fragment counts Delete the document - * Trying reading the doc without timestamp. Verify fragment counts Verify - * read with Point In Time Stamp now. Should be available to read original doc - * within timestamp period. Insert same doc again. Verify read with Point In - * Time Stamp of deletion time. Should not be able ot get document. Verify - * fragment counts second time. - * - * Read again without timestamp. Should be able to read the document. - */ - @Test - public void testBInsertAndDeleteJson() throws KeyManagementException, NoSuchAlgorithmException, IOException, ParserConfigurationException, SAXException, XpathException, - TransformerException - { - System.out.println("Running testBInsertAndDeleteJson"); - // TODO THE TEST. - /* - * From Pubs using XQuery 1) First, create a document: - * xdmp:document-insert("/docs/test.xml", hello)) 2) When you query - * the document, it returns the node you inserted: doc("/docs/test.xml") (: - * returns the node hello :) 3) Delete the document: - * xdmp:document-delete("/docs/test.xml") 4) Query the document again. It - * returns the empty sequence because it was just deleted. 5) Run a - * point-in-time query, specifying the current timestamp. - * xdmp:eval("doc('/docs/test.xml')", (), - * {xdmp:request-timestamp()} ) (: returns - * the empty sequence because the document has been deleted :) 6) Run the - * point-in-time query at one less than the current timestamp, which is the - * old timestamp in this case because only one change has happened to the - * database. xdmp:eval("doc('/docs/test.xml')", (), {xdmp:request-timestamp()-1} - * ) (: returns the deleted version of the document :) - */ - } - - /* - * This test verifies if fragments are available for a document that is - * inserted and then deleted when merge time is set to 60 seconds and both - * insert and delete are within that time period. - * - * 1) With Transaction commit and rollback. Verify within a transaction an - * update and then do a rollback the steps below. 2) With Transaction commit - * and rollback. Verify within a transaction an update and then do a commoi - * the steps below. - * - * Git Issue 457 needs to be completed in order for this test to be fleshed - * completly. - * - * Insert doc Read the doc and Verify fragment counts Delete the document - * Trying reading the doc without timestamp. Verify fragment counts Verify - * read with Point In Time Stamp now. Should be available to read original doc - * within timestamp period. Insert same doc again. Verify read with Point In - * Time Stamp of deletion time. Should not be able ot get document. Verify - * fragment counts second time. - * - * Read again without timestamp. Should be able to read the document. - */ - @Test - public void testCWithTransaction() throws KeyManagementException, NoSuchAlgorithmException, IOException, ParserConfigurationException, SAXException, XpathException, - TransformerException - { - System.out.println("Running testCWithTransaction"); - // TODO THE TEST. - } - - /* - * This test verifies if fragments are available for a document that is - * inserted and then deleted when merge time is set to 60 seconds and both - * insert and delete are within that time period. - * - * 1) With Server side Transformation. - * - * - * Git Issue 457 needs to be completed in order for this test to be fleshed - * completly. - * - * Insert doc Read the doc and Verify fragment counts Delete the document - * Trying reading the doc without timestamp. Verify fragment counts Verify - * read with Point In Time Stamp now. Should be available to read original doc - * within timestamp period. Insert same doc again. Verify read with Point In - * Time Stamp of deletion time. Should not be able ot get document. Verify - * fragment counts second time. - * - * Read again without timestamp. Should be able to read the document. - */ - @Test - public void testDWithTransformation() throws KeyManagementException, NoSuchAlgorithmException, IOException, ParserConfigurationException, SAXException, XpathException, - TransformerException - { - System.out.println("Running testDWithTransformation"); - // TODO THE TEST. - } - - /* - * This test verifies if fragments are available for a document that is - * inserted and then deleted when merge time is set to 60 seconds and both - * insert and delete are within that time period. - * - * 1) Verify results on DocumentPage.search and other methods that take a - * PointInTime parameter. - * - * - * Git Issue 457 needs to be completed in order for this test to be fleshed - * completly. - * - * Insert doc Read the doc and Verify fragment counts Delete the document - * Trying reading the doc without timestamp. Verify fragment counts Verify - * read with Point In Time Stamp now. Should be available to read original doc - * within timestamp period. Insert same doc again. Verify read with Point In - * Time Stamp of deletion time. Should not be able ot get document. Verify - * fragment counts second time. - * - * Read again without timestamp. Should be able to read the document. - */ - @Test - public void testEDocumentPage() throws KeyManagementException, NoSuchAlgorithmException, IOException, ParserConfigurationException, SAXException, XpathException, - TransformerException - { - System.out.println("Running testEDocumentPage"); - // TODO THE TEST. - } - - static class DatabaseCounts { - public int activeFragments; - public int deletedFragments; - - @Override - public String toString() { - return "active=" + activeFragments + "; deleted=" + deletedFragments; - } - } - - public DatabaseCounts getDBFragmentCount() { - DefaultHttpClient client = null; - try { - client = new DefaultHttpClient(); - client.getCredentialsProvider().setCredentials( - new AuthScope(appServerHostname, adminPort), - new UsernamePasswordCredentials("admin", "admin")); - String url = "http://" + appServerHostname + ":" + adminPort + "/manage/v2/databases/" + dbName + "?view=counts&format=xml"; - HttpGet getrequest = new HttpGet(url); - HttpResponse resp = client.execute(getrequest); - Document doc = new SAXBuilder().build(resp.getEntity().getContent()); - Namespace ns = Namespace.getNamespace("db", "http://marklogic.com/manage/databases"); - Element counts = doc.getRootElement().getChild("count-properties", ns); - DatabaseCounts dbCounts = new DatabaseCounts(); - dbCounts.activeFragments = Integer.parseInt(counts.getChildText("active-fragments", ns)); - dbCounts.deletedFragments = Integer.parseInt(counts.getChildText("deleted-fragments", ns)); - return dbCounts; - } catch (Exception e) { - throw new RuntimeException(e); - } finally { - client.getConnectionManager().shutdown(); - } - } - - @AfterAll - public static void tearDown() throws Exception - { - System.out.println("In tear down"); - cleanupRESTServer(dbName, fNames); - deleteRESTUser("eval-user"); - deleteUserRole("test-eval"); - } -} diff --git a/marklogic-client-api/src/test/java/com/marklogic/client/test/datamovement/LegalHoldsTest.java b/marklogic-client-api/src/test/java/com/marklogic/client/test/datamovement/LegalHoldsTest.java index ca4d058a2..1d48a2c0c 100644 --- a/marklogic-client-api/src/test/java/com/marklogic/client/test/datamovement/LegalHoldsTest.java +++ b/marklogic-client-api/src/test/java/com/marklogic/client/test/datamovement/LegalHoldsTest.java @@ -208,16 +208,6 @@ private static void installModule() throws Exception { .withFormat(Format.TEXT)); } - private static void setMergeTimestamp(String timestampXQuery) throws Exception { - restAdminClient.newServerEval() - .xquery( - "import module namespace admin = 'http://marklogic.com/xdmp/admin' at '/MarkLogic/admin.xqy';" + - - "admin:save-configuration(admin:database-set-merge-timestamp(admin:get-configuration(), " + - " xdmp:database(), " + timestampXQuery + ")), " + timestampXQuery - ).eval(); - } - private static void uploadData() throws Exception { DocumentMetadataHandle meta = new DocumentMetadataHandle() .withCollections(collection); diff --git a/test-app/src/main/ml-config/databases/func-content-database.json b/test-app/src/main/ml-config/databases/func-content-database.json index 6558999fe..28b4ef38a 100644 --- a/test-app/src/main/ml-config/databases/func-content-database.json +++ b/test-app/src/main/ml-config/databases/func-content-database.json @@ -207,8 +207,40 @@ "collation": "http://marklogic.com/collation/", "range-value-positions": false, "invalid-values": "reject" - } - ], + }, + { + "scalar-type": "int", + "namespace-uri": "", + "localname": "int", + "collation": "", + "range-value-positions": false, + "invalid-values": "ignore" + }, + { + "scalar-type": "string", + "namespace-uri": "", + "localname": "grandchild", + "collation": "http://marklogic.com/collation/", + "range-value-positions": false, + "invalid-values": "reject" + }, + { + "scalar-type": "string", + "namespace-uri": "", + "localname": "string", + "collation": "http://marklogic.com/collation/", + "range-value-positions": false, + "invalid-values": "reject" + }, + { + "scalar-type": "double", + "namespace-uri": "", + "localname": "double", + "collation": "", + "range-value-positions": false, + "invalid-values": "reject" + } + ], "element-word-lexicon": [ { "namespace-uri": "", @@ -505,4 +537,4 @@ "invalid-values": "reject" } ] -} \ No newline at end of file +}