From 4ddb72802eb9abcb4952c560382844ab4b34351f Mon Sep 17 00:00:00 2001 From: Al Niessner Date: Mon, 29 Apr 2024 12:23:18 -0700 Subject: [PATCH] clean up bad references --- pom.xml | 2 +- .../registry/mgr/cmd/data/DeleteDataCmd.java | 105 ++---- .../registry/mgr/cmd/data/ExportFileCmd.java | 105 ++---- .../mgr/cmd/data/UpdateAltIdsCmd.java | 18 +- .../pds/registry/mgr/cmd/dd/DeleteDDCmd.java | 121 ++----- .../pds/registry/mgr/cmd/dd/ListDDCmd.java | 10 +- .../pds/registry/mgr/cmd/dd/LoadDDCmd.java | 22 +- .../pds/registry/mgr/dao/DataExporter.java | 100 ++---- .../mgr/{util/es => dao}/EsDocWriter.java | 29 +- .../pds/registry/mgr/dao/RegistryDao.java | 48 ++- .../mgr/dao/RegistryDataExporter.java | 20 +- .../pds/registry/mgr/dao/RegistryManager.java | 15 +- .../mgr/dao/RegistryRequestBuilder.java | 317 ------------------ .../registry/mgr/dao/dd/DDDataExporter.java | 14 +- .../pds/registry/mgr/srv/IndexService.java | 7 +- .../registry/mgr/util/es/EsQueryUtils.java | 97 ------ src/test/java/tools/LoadLdds.java | 9 +- src/test/java/tt/TestDDRequestBuilder.java | 19 -- src/test/java/tt/TestEsQueryBuilder.java | 33 -- src/test/java/tt/TestLddLoader.java | 8 +- src/test/java/tt/TestRegistryRequestBld.java | 46 --- 21 files changed, 209 insertions(+), 936 deletions(-) rename src/main/java/gov/nasa/pds/registry/mgr/{util/es => dao}/EsDocWriter.java (77%) delete mode 100644 src/main/java/gov/nasa/pds/registry/mgr/dao/RegistryRequestBuilder.java delete mode 100644 src/main/java/gov/nasa/pds/registry/mgr/util/es/EsQueryUtils.java delete mode 100644 src/test/java/tt/TestDDRequestBuilder.java delete mode 100644 src/test/java/tt/TestEsQueryBuilder.java delete mode 100644 src/test/java/tt/TestRegistryRequestBld.java diff --git a/pom.xml b/pom.xml index 6be0b38..b2a1662 100644 --- a/pom.xml +++ b/pom.xml @@ -78,7 +78,7 @@ POSSIBILITY OF SUCH DAMAGE. gov.nasa.pds registry-common - 1.6.0 + 1.6.0-SNAPSHOT diff --git a/src/main/java/gov/nasa/pds/registry/mgr/cmd/data/DeleteDataCmd.java b/src/main/java/gov/nasa/pds/registry/mgr/cmd/data/DeleteDataCmd.java index db19816..7a19ad2 100644 --- a/src/main/java/gov/nasa/pds/registry/mgr/cmd/data/DeleteDataCmd.java +++ b/src/main/java/gov/nasa/pds/registry/mgr/cmd/data/DeleteDataCmd.java @@ -1,24 +1,14 @@ package gov.nasa.pds.registry.mgr.cmd.data; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.io.Reader; -import java.util.Map; import org.apache.commons.cli.CommandLine; -import org.elasticsearch.client.Request; -import org.elasticsearch.client.Response; -import org.elasticsearch.client.ResponseException; -import org.elasticsearch.client.RestClient; - -import com.google.gson.Gson; - -import gov.nasa.pds.registry.common.es.client.EsClientFactory; -import gov.nasa.pds.registry.common.es.client.EsUtils; -import gov.nasa.pds.registry.common.util.CloseUtils; +import gov.nasa.pds.registry.common.ConnectionFactory; +import gov.nasa.pds.registry.common.EstablishConnectionFactory; +import gov.nasa.pds.registry.common.Request; +import gov.nasa.pds.registry.common.ResponseException; +import gov.nasa.pds.registry.common.RestClient; import gov.nasa.pds.registry.mgr.Constants; import gov.nasa.pds.registry.mgr.cmd.CliCommand; -import gov.nasa.pds.registry.mgr.dao.RegistryRequestBuilder; /** @@ -30,8 +20,6 @@ public class DeleteDataCmd implements CliCommand { private String filterMessage; - private String regQuery; - private String refsQuery; /** @@ -55,88 +43,48 @@ public void run(CommandLine cmdLine) throws Exception String indexName = cmdLine.getOptionValue("index", Constants.DEFAULT_REGISTRY_INDEX); String authPath = cmdLine.getOptionValue("auth"); - buildEsQuery(cmdLine); System.out.println("Elasticsearch URL: " + esUrl); System.out.println(" Index: " + indexName); System.out.println(filterMessage); System.out.println(); - - RestClient client = null; - - try - { - client = EsClientFactory.createRestClient(esUrl, authPath); + + ConnectionFactory conFact = EstablishConnectionFactory.from(esUrl, authPath); + try (RestClient client = conFact.createRestClient()) { + Request.DeleteByQuery regQuery = client.createDeleteByQuery().setIndex(indexName), + refQuery = client.createDeleteByQuery().setIndex(indexName + "-refs"); + buildEsQuery(cmdLine, regQuery, refQuery); // Delete from registry index - deleteByQuery(client, indexName, regQuery); + deleteByQuery(indexName, client.performRequest(regQuery)); // Delete from product references index - deleteByQuery(client, indexName + "-refs", refsQuery); + deleteByQuery(indexName + "-refs", client.performRequest(refQuery)); } catch(ResponseException ex) { - throw new Exception(EsUtils.extractErrorMessage(ex)); - } - finally - { - CloseUtils.close(client); + throw new Exception(ex.extractErrorMessage()); } } - private static void deleteByQuery(RestClient client, String indexName, String query) throws Exception - { - Request req = new Request("POST", "/" + indexName + "/_delete_by_query"); - req.setJsonEntity(query); - - Response resp = client.performRequest(req); - double numDeleted = extractNumDeleted(resp); - - System.out.format("Deleted %.0f document(s) from %s index\n", numDeleted, indexName); - } - - - @SuppressWarnings("rawtypes") - private static double extractNumDeleted(Response resp) + private static void deleteByQuery(String indexName, long numDeleted) throws Exception { - try - { - InputStream is = resp.getEntity().getContent(); - Reader rd = new InputStreamReader(is); - - Gson gson = new Gson(); - Object obj = gson.fromJson(rd, Object.class); - rd.close(); - - obj = ((Map)obj).get("deleted"); - return (Double)obj; - } - catch(Exception ex) - { - return 0; - } + System.out.format("Deleted %d document(s) from %s index\n", numDeleted, indexName); } - /** * Build Elasticsearch query to delete records. * Records can be deleted by LIDVID, LID, PackageID. All records can also be deleted. * @param cmdLine * @throws Exception */ - private void buildEsQuery(CommandLine cmdLine) throws Exception - { - // Registry index - RegistryRequestBuilder regBld = new RegistryRequestBuilder(); - // Product references index - RegistryRequestBuilder refsBld = new RegistryRequestBuilder(); - + private void buildEsQuery(CommandLine cmdLine, Request.DeleteByQuery regQuery, Request.DeleteByQuery refsQuery) throws Exception + { String id = cmdLine.getOptionValue("lidvid"); if(id != null) { this.filterMessage = " LIDVID: " + id; - this.regQuery = regBld.createFilterQuery("lidvid", id); - this.refsQuery = refsBld.createFilterQuery("collection_lidvid", id); - + regQuery.createFilterQuery("lidvid", id); + refsQuery.createFilterQuery("collection_lidvid", id); return; } @@ -144,9 +92,8 @@ private void buildEsQuery(CommandLine cmdLine) throws Exception if(id != null) { this.filterMessage = " LID: " + id; - this.regQuery = regBld.createFilterQuery("lid", id); - this.refsQuery = refsBld.createFilterQuery("collection_lid", id); - + regQuery.createFilterQuery("lid", id); + refsQuery.createFilterQuery("collection_lid", id); return; } @@ -154,8 +101,8 @@ private void buildEsQuery(CommandLine cmdLine) throws Exception if(id != null) { this.filterMessage = " Package ID: " + id; - this.regQuery = regBld.createFilterQuery("_package_id", id); - this.refsQuery = refsBld.createFilterQuery("_package_id", id); + regQuery.createFilterQuery("_package_id", id); + refsQuery.createFilterQuery("_package_id", id); return; } @@ -163,8 +110,8 @@ private void buildEsQuery(CommandLine cmdLine) throws Exception if(cmdLine.hasOption("all")) { this.filterMessage = "Delete all documents "; - this.regQuery = regBld.createMatchAllQuery(); - this.refsQuery = refsBld.createMatchAllQuery(); + regQuery.createMatchAllQuery(); + refsQuery.createMatchAllQuery(); return; } diff --git a/src/main/java/gov/nasa/pds/registry/mgr/cmd/data/ExportFileCmd.java b/src/main/java/gov/nasa/pds/registry/mgr/cmd/data/ExportFileCmd.java index a140209..a1b7d4b 100644 --- a/src/main/java/gov/nasa/pds/registry/mgr/cmd/data/ExportFileCmd.java +++ b/src/main/java/gov/nasa/pds/registry/mgr/cmd/data/ExportFileCmd.java @@ -1,20 +1,13 @@ package gov.nasa.pds.registry.mgr.cmd.data; -import java.util.Map; - import org.apache.commons.cli.CommandLine; -import org.elasticsearch.client.Request; -import org.elasticsearch.client.Response; -import org.elasticsearch.client.ResponseException; -import org.elasticsearch.client.RestClient; - -import gov.nasa.pds.registry.common.es.client.EsClientFactory; -import gov.nasa.pds.registry.common.es.client.EsUtils; -import gov.nasa.pds.registry.common.es.client.SearchResponseParser; -import gov.nasa.pds.registry.common.util.CloseUtils; +import gov.nasa.pds.registry.common.ConnectionFactory; +import gov.nasa.pds.registry.common.EstablishConnectionFactory; +import gov.nasa.pds.registry.common.Request; +import gov.nasa.pds.registry.common.ResponseException; +import gov.nasa.pds.registry.common.RestClient; import gov.nasa.pds.registry.mgr.Constants; import gov.nasa.pds.registry.mgr.cmd.CliCommand; -import gov.nasa.pds.registry.mgr.dao.RegistryRequestBuilder; import gov.nasa.pds.registry.mgr.util.EmbeddedBlobExporter; @@ -30,50 +23,6 @@ public class ExportFileCmd implements CliCommand * * @author karpenko */ - private static class ResponseCB implements SearchResponseParser.Callback - { - private boolean found = false; - private String lidvid; - private String filePath; - - /** - * Constructor - * @param lidvid LidVid of a document with BLOB - * @param filePath File path to export BLOB to. - */ - public ResponseCB(String lidvid, String filePath) - { - this.lidvid = lidvid; - this.filePath = filePath; - } - - - @Override - @SuppressWarnings("rawtypes") - public void onRecord(String id, Object rec) throws Exception - { - found = true; - - Object blob = ((Map)rec).get(Constants.BLOB_FIELD); - if(blob == null) - { - System.out.println("There is no BLOB in a document with LIDVID = " + lidvid); - System.out.println("Probably embedded BLOB storage was not enabled when the document was created."); - return; - } - - EmbeddedBlobExporter.export(blob.toString(), filePath); - System.out.println("Done"); - } - - - public boolean found() - { - return found; - } - - } - /** * Constructor @@ -116,39 +65,27 @@ public void run(CommandLine cmdLine) throws Exception System.out.println(" Output file: " + filePath); System.out.println(); - RestClient client = null; - - try + ConnectionFactory conFact = EstablishConnectionFactory.from(esUrl, authPath); + try (RestClient client = conFact.createRestClient()) { - // Create Elasticsearch client - client = EsClientFactory.createRestClient(esUrl, authPath); - - // Create request - Request req = new Request("GET", "/" + indexName + "/_search"); - RegistryRequestBuilder bld = new RegistryRequestBuilder(); - String jsonReq = bld.createGetBlobRequest(lidvid); - req.setJsonEntity(jsonReq); - - // Execute request - Response resp = client.performRequest(req); + Request.Search req = client.createSearchRequest().setIndex(indexName).buildGetField(Constants.BLOB_FIELD, lidvid); + String blob = client.performRequest(req).field(Constants.BLOB_FIELD); + if(blob == null) + { + System.out.println("There is no BLOB in a document with LIDVID = " + lidvid); + System.out.println("Probably embedded BLOB storage was not enabled when the document was created."); + return; + } - SearchResponseParser respParser = new SearchResponseParser(); - ResponseCB cb = new ResponseCB(lidvid, filePath); - respParser.parseResponse(resp, cb); - - if(!cb.found()) - { - System.out.println("Could not find a document with lidvid = " + lidvid); - return; - } + EmbeddedBlobExporter.export(blob, filePath); + System.out.println("Done"); + } + catch (NoSuchFieldException e) { + System.out.println("No documents found matching lidvid: " + lidvid); } catch(ResponseException ex) { - throw new Exception(EsUtils.extractErrorMessage(ex)); - } - finally - { - CloseUtils.close(client); + throw new Exception(ex.extractErrorMessage()); } } diff --git a/src/main/java/gov/nasa/pds/registry/mgr/cmd/data/UpdateAltIdsCmd.java b/src/main/java/gov/nasa/pds/registry/mgr/cmd/data/UpdateAltIdsCmd.java index 3708d8f..c549bc1 100644 --- a/src/main/java/gov/nasa/pds/registry/mgr/cmd/data/UpdateAltIdsCmd.java +++ b/src/main/java/gov/nasa/pds/registry/mgr/cmd/data/UpdateAltIdsCmd.java @@ -14,10 +14,7 @@ import org.apache.commons.lang3.StringUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.elasticsearch.client.ResponseException; - -import gov.nasa.pds.registry.common.cfg.RegistryCfg; -import gov.nasa.pds.registry.common.es.client.EsUtils; +import gov.nasa.pds.registry.common.ResponseException; import gov.nasa.pds.registry.common.util.CloseUtils; import gov.nasa.pds.registry.mgr.Constants; import gov.nasa.pds.registry.mgr.cmd.CliCommand; @@ -51,12 +48,9 @@ public void run(CommandLine cmdLine) throws Exception printHelp(); return; } - - RegistryCfg cfg = new RegistryCfg(); - cfg.url = cmdLine.getOptionValue("es", "app:/connections/direct/localhost.xml"); - cfg.indexName = cmdLine.getOptionValue("index", Constants.DEFAULT_REGISTRY_INDEX); - cfg.authFile = cmdLine.getOptionValue("auth"); - + String url = cmdLine.getOptionValue("es", "app:/connections/direct/localhost.xml"); + String indexName = cmdLine.getOptionValue("index", Constants.DEFAULT_REGISTRY_INDEX); + String authFile = cmdLine.getOptionValue("auth"); String filePath = cmdLine.getOptionValue("file"); if(filePath == null) throw new Exception("Missing required parameter '-file'"); File file = new File(filePath); @@ -64,12 +58,12 @@ public void run(CommandLine cmdLine) throws Exception try { - RegistryManager.init(cfg); + RegistryManager.init(url, authFile, indexName); updateIds(file); } catch(ResponseException ex) { - throw new Exception(EsUtils.extractErrorMessage(ex)); + throw new Exception(ex.extractErrorMessage()); } finally { diff --git a/src/main/java/gov/nasa/pds/registry/mgr/cmd/dd/DeleteDDCmd.java b/src/main/java/gov/nasa/pds/registry/mgr/cmd/dd/DeleteDDCmd.java index 865af01..7127e78 100644 --- a/src/main/java/gov/nasa/pds/registry/mgr/cmd/dd/DeleteDDCmd.java +++ b/src/main/java/gov/nasa/pds/registry/mgr/cmd/dd/DeleteDDCmd.java @@ -1,24 +1,14 @@ package gov.nasa.pds.registry.mgr.cmd.dd; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.io.Reader; -import java.util.Map; - import org.apache.commons.cli.CommandLine; -import org.elasticsearch.client.Request; -import org.elasticsearch.client.Response; -import org.elasticsearch.client.ResponseException; -import org.elasticsearch.client.RestClient; - -import com.google.gson.Gson; -import gov.nasa.pds.registry.common.es.client.EsClientFactory; -import gov.nasa.pds.registry.common.es.client.EsUtils; -import gov.nasa.pds.registry.common.util.CloseUtils; +import gov.nasa.pds.registry.common.ConnectionFactory; +import gov.nasa.pds.registry.common.EstablishConnectionFactory; +import gov.nasa.pds.registry.common.Request; +import gov.nasa.pds.registry.common.ResponseException; +import gov.nasa.pds.registry.common.RestClient; import gov.nasa.pds.registry.mgr.Constants; import gov.nasa.pds.registry.mgr.cmd.CliCommand; -import gov.nasa.pds.registry.mgr.dao.RegistryRequestBuilder; /** @@ -29,6 +19,7 @@ */ public class DeleteDDCmd implements CliCommand { + private ConnectionFactory conFact; private String esUrl; private String indexName; private String authPath; @@ -53,7 +44,7 @@ public void run(CommandLine cmdLine) throws Exception esUrl = cmdLine.getOptionValue("es", "app:/connections/direct/localhost.xml"); indexName = cmdLine.getOptionValue("index", Constants.DEFAULT_REGISTRY_INDEX); authPath = cmdLine.getOptionValue("auth"); - + this.conFact = EstablishConnectionFactory.from(esUrl, authPath).setIndexName(indexName); String id = cmdLine.getOptionValue("id"); if(id != null) { @@ -73,112 +64,42 @@ public void run(CommandLine cmdLine) throws Exception private void deleteById(String id) throws Exception - { - RestClient client = null; - - try + { + try (RestClient client = this.conFact.createRestClient()) { - // Create Elasticsearch client - client = EsClientFactory.createRestClient(esUrl, authPath); - - // Create request - RegistryRequestBuilder bld = new RegistryRequestBuilder(); - String query = bld.createFilterQuery("_id", id); - - Request req = new Request("POST", "/" + indexName + "-dd" + "/_delete_by_query?refresh=true"); - req.setJsonEntity(query); + Request.DeleteByQuery request = client.createDeleteByQuery().setIndex(this.conFact.getIndexName() + "-dd").setRefresh(true).createFilterQuery("_id", id); // Execute request - Response resp = client.performRequest(req); - double numDeleted = extractNumDeleted(resp); + long numDeleted = client.performRequest(request); - System.out.format("Deleted %.0f document(s)\n", numDeleted); + System.out.format("Deleted %d document(s)\n", numDeleted); } catch(ResponseException ex) { - throw new Exception(EsUtils.extractErrorMessage(ex)); + throw new Exception(ex.extractErrorMessage()); } - finally - { - CloseUtils.close(client); - } } private void deleteByNamespace(String ns) throws Exception - { - RestClient client = null; - - try + { + try (RestClient client = this.conFact.createRestClient()) { - // Create Elasticsearch client - client = EsClientFactory.createRestClient(esUrl, authPath); - // (1) Delete by class namespace - - // Create request - RegistryRequestBuilder bld = new RegistryRequestBuilder(); - String query = bld.createFilterQuery("class_ns", ns); - - Request req = new Request("POST", "/" + indexName + "-dd" + "/_delete_by_query?refresh=true"); - req.setJsonEntity(query); - - // Execute request - Response resp = client.performRequest(req); - double numDeleted = extractNumDeleted(resp); + Request.DeleteByQuery request = client.createDeleteByQuery().setIndex(this.conFact.getIndexName() + "-dd").setRefresh(true).createFilterQuery("class_ns", ns); + long numDeleted = client.performRequest(request); // (2) Delete by attribute namespace - - // Create request - query = bld.createFilterQuery("attr_ns", ns); - - req = new Request("POST", "/" + indexName + "-dd" + "/_delete_by_query?refresh=true"); - req.setJsonEntity(query); - - // Execute request - resp = client.performRequest(req); - numDeleted += extractNumDeleted(resp); - - System.out.format("Deleted %.0f document(s)\n", numDeleted); + request = client.createDeleteByQuery().setIndex(this.conFact.getIndexName() + "-dd").setRefresh(true).createFilterQuery("class_ns", ns); + numDeleted += client.performRequest(request); + System.out.format("Deleted %d document(s)\n", numDeleted); } catch(ResponseException ex) { - throw new Exception(EsUtils.extractErrorMessage(ex)); + throw new Exception(ex.extractErrorMessage()); } - finally - { - CloseUtils.close(client); - } } - - /** - * Extract number of deleted records from Elasticsearch delete API response. - * @param resp - * @return number of deleted records - */ - @SuppressWarnings("rawtypes") - private double extractNumDeleted(Response resp) - { - try - { - InputStream is = resp.getEntity().getContent(); - Reader rd = new InputStreamReader(is); - - Gson gson = new Gson(); - Object obj = gson.fromJson(rd, Object.class); - rd.close(); - - obj = ((Map)obj).get("deleted"); - return (Double)obj; - } - catch(Exception ex) - { - return 0; - } - } - - /** * Print help screen */ diff --git a/src/main/java/gov/nasa/pds/registry/mgr/cmd/dd/ListDDCmd.java b/src/main/java/gov/nasa/pds/registry/mgr/cmd/dd/ListDDCmd.java index fdcb2ef..ecd1c2a 100644 --- a/src/main/java/gov/nasa/pds/registry/mgr/cmd/dd/ListDDCmd.java +++ b/src/main/java/gov/nasa/pds/registry/mgr/cmd/dd/ListDDCmd.java @@ -5,7 +5,6 @@ import org.apache.commons.cli.CommandLine; -import gov.nasa.pds.registry.common.cfg.RegistryCfg; import gov.nasa.pds.registry.common.es.dao.dd.DataDictionaryDao; import gov.nasa.pds.registry.common.es.dao.dd.LddInfo; import gov.nasa.pds.registry.mgr.Constants; @@ -36,16 +35,15 @@ public void run(CommandLine cmdLine) throws Exception return; } - RegistryCfg cfg = new RegistryCfg(); - cfg.url = cmdLine.getOptionValue("es", "app:/connections/direct/localhost.xml"); - cfg.indexName = cmdLine.getOptionValue("index", Constants.DEFAULT_REGISTRY_INDEX); - cfg.authFile = cmdLine.getOptionValue("auth"); + String url = cmdLine.getOptionValue("es", "app:/connections/direct/localhost.xml"); + String indexName = cmdLine.getOptionValue("index", Constants.DEFAULT_REGISTRY_INDEX); + String authFile = cmdLine.getOptionValue("auth"); String namespace = cmdLine.getOptionValue("ns"); try { - RegistryManager.init(cfg); + RegistryManager.init(url, authFile, indexName); DataDictionaryDao dao = RegistryManager.getInstance().getDataDictionaryDao(); List list = dao.listLdds(namespace); diff --git a/src/main/java/gov/nasa/pds/registry/mgr/cmd/dd/LoadDDCmd.java b/src/main/java/gov/nasa/pds/registry/mgr/cmd/dd/LoadDDCmd.java index ce265b6..8bd5939 100644 --- a/src/main/java/gov/nasa/pds/registry/mgr/cmd/dd/LoadDDCmd.java +++ b/src/main/java/gov/nasa/pds/registry/mgr/cmd/dd/LoadDDCmd.java @@ -5,7 +5,6 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import gov.nasa.pds.registry.common.EstablishConnectionFactory; -import gov.nasa.pds.registry.common.cfg.RegistryCfg; import gov.nasa.pds.registry.common.dd.LddUtils; import gov.nasa.pds.registry.common.es.dao.DataLoader; import gov.nasa.pds.registry.common.es.dao.dd.DataDictionaryDao; @@ -27,9 +26,9 @@ */ public class LoadDDCmd implements CliCommand { - private RegistryCfg cfg; - - + private String url; + private String indexName; + private String authFile ; /** * Constructor */ @@ -70,12 +69,11 @@ public void run(CommandLine cmdLine) throws Exception return; } - cfg = new RegistryCfg(); - cfg.url = cmdLine.getOptionValue("es", "app:/connections/direct/localhost.xml"); - cfg.indexName = cmdLine.getOptionValue("index", Constants.DEFAULT_REGISTRY_INDEX); - cfg.authFile = cmdLine.getOptionValue("auth"); + this.url = cmdLine.getOptionValue("es", "app:/connections/direct/localhost.xml"); + this.indexName = cmdLine.getOptionValue("index", Constants.DEFAULT_REGISTRY_INDEX); + this.authFile = cmdLine.getOptionValue("auth"); - RegistryManager.init(cfg); + RegistryManager.init(url, authFile, indexName); try { @@ -131,7 +129,7 @@ private void loadLdd(String path, String namespace) throws Exception // Init LDD loader DataDictionaryDao ddDao = RegistryManager.getInstance().getDataDictionaryDao(); - JsonLddLoader loader = new JsonLddLoader(ddDao, EstablishConnectionFactory.from(cfg.url, cfg.authFile).setIndexName(cfg.indexName)); + JsonLddLoader loader = new JsonLddLoader(ddDao, EstablishConnectionFactory.from(this.url, this.authFile).setIndexName(this.indexName)); loader.loadPds2EsDataTypeMap(LddUtils.getPds2EsDataTypeCfgFile("REGISTRY_MANAGER_HOME")); //Load LDD @@ -150,7 +148,7 @@ private void loadDataDump(String path) throws Exception Logger log = LogManager.getLogger(this.getClass()); log.info("Data dump: " + path); - DataLoader loader = new DataLoader(EstablishConnectionFactory.from(cfg.url, cfg.authFile).setIndexName( cfg.indexName + "-dd")); + DataLoader loader = new DataLoader(EstablishConnectionFactory.from(this.url, this.authFile).setIndexName(this.indexName + "-dd")); loader.loadFile(new File(path)); } @@ -165,7 +163,7 @@ private void loadCsv(String path) throws Exception Logger log = LogManager.getLogger(this.getClass()); log.info("CSV file: " + path); - CsvLddLoader loader = new CsvLddLoader(cfg.url, cfg.indexName, cfg.authFile); + CsvLddLoader loader = new CsvLddLoader(this.url, this.indexName, this.authFile); File lddFile = new File(path); loader.load(lddFile); } diff --git a/src/main/java/gov/nasa/pds/registry/mgr/dao/DataExporter.java b/src/main/java/gov/nasa/pds/registry/mgr/dao/DataExporter.java index 954a5bd..2e4df11 100644 --- a/src/main/java/gov/nasa/pds/registry/mgr/dao/DataExporter.java +++ b/src/main/java/gov/nasa/pds/registry/mgr/dao/DataExporter.java @@ -4,16 +4,12 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.elasticsearch.client.Request; -import org.elasticsearch.client.Response; -import org.elasticsearch.client.ResponseException; -import org.elasticsearch.client.RestClient; - -import gov.nasa.pds.registry.common.es.client.EsClientFactory; -import gov.nasa.pds.registry.common.es.client.EsUtils; -import gov.nasa.pds.registry.common.es.client.SearchResponseParser; -import gov.nasa.pds.registry.common.util.CloseUtils; -import gov.nasa.pds.registry.mgr.util.es.EsDocWriter; +import gov.nasa.pds.registry.common.ConnectionFactory; +import gov.nasa.pds.registry.common.EstablishConnectionFactory; +import gov.nasa.pds.registry.common.Request; +import gov.nasa.pds.registry.common.Response; +import gov.nasa.pds.registry.common.ResponseException; +import gov.nasa.pds.registry.common.RestClient; /** @@ -59,7 +55,7 @@ public DataExporter(String esUrl, String indexName, String authConfigFile) * @return JSON * @throws Exception an exception */ - protected abstract String createRequest(int batchSize, String searchAfter) throws Exception; + protected abstract Request.Search createRequest(Request.Search req, int batchSize, String searchAfter); /** @@ -67,60 +63,32 @@ public DataExporter(String esUrl, String indexName, String authConfigFile) * @param file a file * @throws Exception an exception */ - public void export(File file) throws Exception - { - EsDocWriter writer = null; - RestClient client = null; - - try - { - writer = new EsDocWriter(file); - client = EsClientFactory.createRestClient(esUrl, authConfigFile); - SearchResponseParser parser = new SearchResponseParser(); - - String searchAfter = null; - int numDocs = 0; - - do - { - Request req = new Request("GET", "/" + indexName + "/_search"); - // Call abstract method to get JSON query - String json = createRequest(BATCH_SIZE, searchAfter); - req.setJsonEntity(json); - - Response resp = client.performRequest(req); - parser.parseResponse(resp, writer); - - numDocs += parser.getNumDocs(); - searchAfter = parser.getLastId(); - - if(numDocs % PRINT_STATUS_SIZE == 0) - { - log.info("Exported " + numDocs + " document(s)"); - } - } - while(parser.getNumDocs() == BATCH_SIZE); - - if(numDocs == 0) - { - log.info("No documents found"); - } - else - { - log.info("Exported " + numDocs + " document(s)"); - } - - log.info("Done"); + public void export(File file) throws Exception { + ConnectionFactory conFact = EstablishConnectionFactory.from(this.esUrl, this.authConfigFile) + .setIndexName(this.indexName); + try (RestClient client = conFact.createRestClient(); + EsDocWriter writer = new EsDocWriter(file)) { + String searchAfter = null; + int numDocs = 0, thisBatchSize; + do { + Request.Search req = client.createSearchRequest().setIndex(indexName); + req = createRequest(req, BATCH_SIZE, searchAfter); + Response.Search resp = client.performRequest(req); + thisBatchSize = resp.batch().size(); + numDocs += resp.batch().size(); + searchAfter = ""; // FIXME: needs to be the last ID from the batch() + if (numDocs % PRINT_STATUS_SIZE == 0) { + log.info("Exported " + numDocs + " document(s)"); + } + } while (thisBatchSize == BATCH_SIZE); + if (numDocs == 0) { + log.info("No documents found"); + } else { + log.info("Exported " + numDocs + " document(s)"); } - catch(ResponseException ex) - { - throw new Exception(EsUtils.extractErrorMessage(ex)); - } - finally - { - CloseUtils.close(client); - CloseUtils.close(writer); - } - + log.info("Done"); + } catch (ResponseException ex) { + throw new Exception(ex.extractErrorMessage()); + } } -} + } diff --git a/src/main/java/gov/nasa/pds/registry/mgr/util/es/EsDocWriter.java b/src/main/java/gov/nasa/pds/registry/mgr/dao/EsDocWriter.java similarity index 77% rename from src/main/java/gov/nasa/pds/registry/mgr/util/es/EsDocWriter.java rename to src/main/java/gov/nasa/pds/registry/mgr/dao/EsDocWriter.java index 388beeb..8f58695 100644 --- a/src/main/java/gov/nasa/pds/registry/mgr/util/es/EsDocWriter.java +++ b/src/main/java/gov/nasa/pds/registry/mgr/dao/EsDocWriter.java @@ -1,16 +1,15 @@ -package gov.nasa.pds.registry.mgr.util.es; +package gov.nasa.pds.registry.mgr.dao; import java.io.Closeable; import java.io.File; import java.io.FileWriter; import java.io.IOException; import java.io.StringWriter; - +import java.util.List; +import java.util.Map; import com.google.gson.Gson; import com.google.gson.stream.JsonWriter; -import gov.nasa.pds.registry.common.es.client.SearchResponseParser; - /** *

* Elasticsearch document writer. @@ -27,7 +26,7 @@ * * @author karpenko */ -public class EsDocWriter implements Closeable, SearchResponseParser.Callback +class EsDocWriter implements Closeable { private FileWriter writer; private Gson gson; @@ -53,21 +52,25 @@ public void close() throws IOException writer.close(); } - - /** - * Search response parser callback implementation. - * This method is called for every record. - */ - @Override - public void onRecord(String id, Object rec) throws IOException + @SuppressWarnings("unchecked") + public void write (List batch) { + for (Object doc : batch) { + this.onRecord(((Map)batch).get("lidvid").toString(), doc); + } + } + private void onRecord(String id, Object rec) { // 1st line: ID - writePK(id); + try { + writePK(id); newLine(); // 2nd line: data gson.toJson(rec, writer); newLine(); + } catch (IOException e) { + throw new RuntimeException("Should never make it here"); + } } diff --git a/src/main/java/gov/nasa/pds/registry/mgr/dao/RegistryDao.java b/src/main/java/gov/nasa/pds/registry/mgr/dao/RegistryDao.java index 73df7c0..8a4d8b0 100644 --- a/src/main/java/gov/nasa/pds/registry/mgr/dao/RegistryDao.java +++ b/src/main/java/gov/nasa/pds/registry/mgr/dao/RegistryDao.java @@ -1,8 +1,12 @@ package gov.nasa.pds.registry.mgr.dao; +import java.io.StringWriter; +import java.util.ArrayList; import java.util.Collection; +import java.util.List; import java.util.Map; import java.util.Set; +import com.google.gson.stream.JsonWriter; import gov.nasa.pds.registry.common.Request; import gov.nasa.pds.registry.common.RestClient; import gov.nasa.pds.registry.common.util.Tuple; @@ -68,14 +72,54 @@ public void updateAlternateIds(Map> newIds) throws Exception { if(newIds == null || newIds.isEmpty()) return; - RegistryRequestBuilder bld = new RegistryRequestBuilder(); Request.Bulk req = client.createBulkRequest() .setIndex(this.indexName) .setRefresh(Request.Bulk.Refresh.WaitFor); - for (Tuple t : bld.createUpdateAltIdsRequest(newIds)) { + for (Tuple t : this.createUpdateAltIdsRequest(newIds)) { req.add(t.item1, t.item2); } client.performRequest(req).logErrors(); } + private List createUpdateAltIdsRequest(Map> newIds) throws Exception + { + if(newIds == null || newIds.isEmpty()) throw new IllegalArgumentException("Missing ids"); + ArrayList updates = new ArrayList(); + // Build NJSON (new-line delimited JSON) + for(Map.Entry> entry: newIds.entrySet()) + { + Tuple statement = new Tuple(); + // Line 1: Elasticsearch document ID + statement.item1 = "{ \"update\" : {\"_id\" : \"" + entry.getKey() + "\" } }"; + // Line 2: Data + statement.item2 = buildUpdateDocJson("alternate_ids", entry.getValue()); + updates.add(statement); + } + return updates; + } + private String buildUpdateDocJson(String field, Collection values) throws Exception + { + StringWriter out = new StringWriter(); + JsonWriter writer = new JsonWriter(out);; + + writer.beginObject(); + + writer.name("doc"); + writer.beginObject(); + + writer.name(field); + + writer.beginArray(); + for(String value: values) + { + writer.value(value); + } + writer.endArray(); + + writer.endObject(); + writer.endObject(); + + writer.close(); + return out.toString(); + } } diff --git a/src/main/java/gov/nasa/pds/registry/mgr/dao/RegistryDataExporter.java b/src/main/java/gov/nasa/pds/registry/mgr/dao/RegistryDataExporter.java index 1f1b670..16df0ff 100644 --- a/src/main/java/gov/nasa/pds/registry/mgr/dao/RegistryDataExporter.java +++ b/src/main/java/gov/nasa/pds/registry/mgr/dao/RegistryDataExporter.java @@ -1,5 +1,7 @@ package gov.nasa.pds.registry.mgr.dao; +import gov.nasa.pds.registry.common.Request.Search; + /** * Exports data records from Elasticsearch "registry" index into a file. * @@ -35,20 +37,10 @@ public void setFilterField(String name, String value) this.filterFieldValue = value; } - - /** - * Create Elasticsearch JSON query. - */ - @Override - protected String createRequest(int batchSize, String searchAfter) throws Exception - { - RegistryRequestBuilder reqBld = new RegistryRequestBuilder(); - - String json = (filterFieldName == null) ? - reqBld.createExportAllDataRequest("lidvid", batchSize, searchAfter) : - reqBld.createExportDataRequest(filterFieldName, filterFieldValue, "lidvid", batchSize, searchAfter); - - return json; + @Override + protected Search createRequest(Search req, int batchSize, String searchAfter) { + return filterFieldName == null ? req.all("lidvid", batchSize, searchAfter) : + req.all(this.filterFieldName, this.filterFieldValue, "lidvid", batchSize, searchAfter); } } diff --git a/src/main/java/gov/nasa/pds/registry/mgr/dao/RegistryManager.java b/src/main/java/gov/nasa/pds/registry/mgr/dao/RegistryManager.java index 3a6962a..56c846c 100644 --- a/src/main/java/gov/nasa/pds/registry/mgr/dao/RegistryManager.java +++ b/src/main/java/gov/nasa/pds/registry/mgr/dao/RegistryManager.java @@ -4,7 +4,6 @@ import org.apache.logging.log4j.Logger; import gov.nasa.pds.registry.common.EstablishConnectionFactory; import gov.nasa.pds.registry.common.RestClient; -import gov.nasa.pds.registry.common.cfg.RegistryCfg; import gov.nasa.pds.registry.common.es.dao.dd.DataDictionaryDao; import gov.nasa.pds.registry.common.util.CloseUtils; import gov.nasa.pds.registry.common.es.dao.schema.SchemaDao; @@ -30,20 +29,20 @@ public class RegistryManager * @param cfg Registry (Elasticsearch) configuration parameters. * @throws Exception Generic exception */ - private RegistryManager(RegistryCfg cfg) throws Exception + private RegistryManager(String connURL, String authFile, String index) throws Exception { - if(cfg.url == null || cfg.url.isEmpty()) throw new IllegalArgumentException("Missing Registry URL"); + if(connURL == null || connURL.isEmpty()) throw new IllegalArgumentException("Missing Registry URL"); - client = EstablishConnectionFactory.from(cfg.url, cfg.authFile).createRestClient(); + client = EstablishConnectionFactory.from(connURL, authFile).createRestClient(); - String indexName = cfg.indexName; + String indexName = index; if(indexName == null || indexName.isEmpty()) { indexName = "registry"; } Logger log = LogManager.getLogger(this.getClass()); - log.info("Registry URL: " + cfg.url); + log.info("Registry URL: " + connURL); log.info("Registry index: " + indexName); schemaDao = new SchemaDao(client, indexName); @@ -57,9 +56,9 @@ private RegistryManager(RegistryCfg cfg) throws Exception * @param cfg Registry (Elasticsearch) configuration parameters. * @throws Exception Generic exception */ - public static void init(RegistryCfg cfg) throws Exception + public static void init(String connURL, String authFile, String index) throws Exception { - instance = new RegistryManager(cfg); + instance = new RegistryManager(connURL, authFile, index); } diff --git a/src/main/java/gov/nasa/pds/registry/mgr/dao/RegistryRequestBuilder.java b/src/main/java/gov/nasa/pds/registry/mgr/dao/RegistryRequestBuilder.java deleted file mode 100644 index 9c9f935..0000000 --- a/src/main/java/gov/nasa/pds/registry/mgr/dao/RegistryRequestBuilder.java +++ /dev/null @@ -1,317 +0,0 @@ -package gov.nasa.pds.registry.mgr.dao; - -import java.io.File; -import java.io.FileReader; -import java.io.IOException; -import java.io.StringWriter; -import java.io.Writer; -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.TreeMap; - -import com.google.gson.Gson; -import com.google.gson.stream.JsonWriter; - -import gov.nasa.pds.registry.common.util.CloseUtils; -import gov.nasa.pds.registry.common.util.Tuple; -import gov.nasa.pds.registry.mgr.Constants; -import gov.nasa.pds.registry.mgr.util.es.EsQueryUtils; - -/** - * A class to build Elasticsearch API JSON requests. - * - * @author karpenko - */ -public class RegistryRequestBuilder -{ - private boolean pretty; - - - /** - * Constructor - * @param pretty Pretty-format JSON requests - */ - public RegistryRequestBuilder(boolean pretty) - { - this.pretty = pretty; - } - - - /** - * Constructor - */ - public RegistryRequestBuilder() - { - this(false); - } - - - private JsonWriter createJsonWriter(Writer writer) - { - JsonWriter jw = new JsonWriter(writer); - if (pretty) - { - jw.setIndent(" "); - } - - return jw; - } - - - /** - * Build create index request - * @param schemaFile index schema file - * @param shards number of shards - * @param replicas number of replicas - * @return JSON - * @throws Exception Generic exception - */ - @SuppressWarnings({"rawtypes", "unchecked"}) - public String createCreateIndexRequest(File schemaFile, int shards, int replicas) throws Exception - { - // Read schema template - FileReader rd = new FileReader(schemaFile); - Gson gson = new Gson(); - Object rootObj = gson.fromJson(rd, Object.class); - CloseUtils.close(rd); - - Object settingsObj = ((Map)rootObj).get("settings"); - if (settingsObj == null) - { - settingsObj = new TreeMap(); - } - - Object mappingsObj = ((Map)rootObj).get("mappings"); - if (mappingsObj == null) - { - throw new Exception("Missing mappings in schema file " + schemaFile.getAbsolutePath()); - } - - StringWriter out = new StringWriter(); - JsonWriter writer = createJsonWriter(out); - - writer.beginObject(); - - Map settingsMap = (Map)settingsObj; - settingsMap.put("number_of_shards", shards); - settingsMap.put("number_of_replicas", replicas); - - // Settings - writer.name("settings"); - gson.toJson(settingsObj, Object.class, writer); - - // Mappings - writer.name("mappings"); - gson.toJson(mappingsObj, Object.class, writer); - - writer.endObject(); - - writer.close(); - return out.toString(); - } - - - /** - * Build export data request - * @param filterField Filter field name, such as "lidvid". - * @param filterValue Filter value. - * @param sortField Sort field is required to paginate data and use "search_after" field. - * @param size Batch / page size - * @param searchAfter "search_after" field to perform pagination - * @return JSON - * @throws IOException an exception - */ - public String createExportDataRequest(String filterField, String filterValue, - String sortField, int size, String searchAfter) throws IOException - { - StringWriter out = new StringWriter(); - JsonWriter writer = createJsonWriter(out); - - writer.beginObject(); - - // Size (number of records to return) - writer.name("size").value(size); - - // Filter query - EsQueryUtils.appendFilterQuery(writer, filterField, filterValue); - - // "search_after" parameter is used for pagination - if (searchAfter != null) - { - writer.name("search_after").value(searchAfter); - } - - // Sort is required by pagination - writer.name("sort"); - writer.beginObject(); - writer.name(sortField).value("asc"); - writer.endObject(); - - writer.endObject(); - - writer.close(); - return out.toString(); - } - - - /** - * Build export all data request - * @param sortField Sort field is required to paginate data and use "search_after" field. - * @param size Batch / page size - * @param searchAfter "search_after" field to perform pagination - * @return JSON - * @throws IOException an exception - */ - public String createExportAllDataRequest(String sortField, int size, String searchAfter) throws IOException - { - StringWriter out = new StringWriter(); - JsonWriter writer = createJsonWriter(out); - - writer.beginObject(); - - // Size (number of records to return) - writer.name("size").value(size); - - // Match all query - EsQueryUtils.appendMatchAllQuery(writer); - - // "search_after" parameter is used for pagination - if (searchAfter != null) - { - writer.name("search_after"); - writer.beginArray(); - writer.value(searchAfter); - writer.endArray(); - } - - // Sort is required by pagination - writer.name("sort"); - writer.beginObject(); - writer.name(sortField).value("asc"); - writer.endObject(); - - writer.endObject(); - - writer.close(); - return out.toString(); - } - - - /** - * Build get BLOB request - * @param lidvid a LidVid - * @return JSON - * @throws IOException an exception - */ - public String createGetBlobRequest(String lidvid) throws IOException - { - StringWriter out = new StringWriter(); - JsonWriter writer = createJsonWriter(out); - - writer.beginObject(); - - // Return only BLOB - writer.name("_source"); - writer.beginArray(); - writer.value(Constants.BLOB_FIELD); - writer.endArray(); - - // Query - EsQueryUtils.appendFilterQuery(writer, "lidvid", lidvid); - writer.endObject(); - - writer.close(); - return out.toString(); - } - - - /** - * Create Elasticsearch filter query - * @param field filter field name - * @param value filter value - * @return JSON - * @throws IOException an exception - */ - public String createFilterQuery(String field, String value) throws IOException - { - StringWriter out = new StringWriter(); - JsonWriter writer = createJsonWriter(out); - - writer.beginObject(); - EsQueryUtils.appendFilterQuery(writer, field, value); - writer.endObject(); - - writer.close(); - return out.toString(); - } - - - /** - * Build match all query - * @return JSON - * @throws IOException an exception - */ - public String createMatchAllQuery() throws IOException - { - StringWriter out = new StringWriter(); - JsonWriter writer = createJsonWriter(out); - - writer.beginObject(); - - writer.name("query"); - writer.beginObject(); - EsQueryUtils.appendMatchAll(writer); - writer.endObject(); - - writer.endObject(); - - writer.close(); - return out.toString(); - } - public List createUpdateAltIdsRequest(Map> newIds) throws Exception - { - if(newIds == null || newIds.isEmpty()) throw new IllegalArgumentException("Missing ids"); - ArrayList updates = new ArrayList(); - // Build NJSON (new-line delimited JSON) - for(Map.Entry> entry: newIds.entrySet()) - { - Tuple statement = new Tuple(); - // Line 1: Elasticsearch document ID - statement.item1 = "{ \"update\" : {\"_id\" : \"" + entry.getKey() + "\" } }"; - // Line 2: Data - statement.item2 = buildUpdateDocJson("alternate_ids", entry.getValue()); - updates.add(statement); - } - return updates; - } - - - private String buildUpdateDocJson(String field, Collection values) throws Exception - { - StringWriter out = new StringWriter(); - JsonWriter writer = createJsonWriter(out); - - writer.beginObject(); - - writer.name("doc"); - writer.beginObject(); - - writer.name(field); - - writer.beginArray(); - for(String value: values) - { - writer.value(value); - } - writer.endArray(); - - writer.endObject(); - writer.endObject(); - - writer.close(); - return out.toString(); - } -} diff --git a/src/main/java/gov/nasa/pds/registry/mgr/dao/dd/DDDataExporter.java b/src/main/java/gov/nasa/pds/registry/mgr/dao/dd/DDDataExporter.java index ebc0ef1..7b5756f 100644 --- a/src/main/java/gov/nasa/pds/registry/mgr/dao/dd/DDDataExporter.java +++ b/src/main/java/gov/nasa/pds/registry/mgr/dao/dd/DDDataExporter.java @@ -1,7 +1,7 @@ package gov.nasa.pds.registry.mgr.dao.dd; +import gov.nasa.pds.registry.common.Request.Search; import gov.nasa.pds.registry.mgr.dao.DataExporter; -import gov.nasa.pds.registry.mgr.dao.RegistryRequestBuilder; /** * Exports data dictionary records from Elasticsearch into a file. @@ -20,17 +20,9 @@ public DDDataExporter(String esUrl, String indexName, String authConfigFile) { super(esUrl, indexName + "-dd", authConfigFile); } - - - /** - * Creates Elasticsearch JSON query. - */ @Override - protected String createRequest(int batchSize, String searchAfter) throws Exception - { - RegistryRequestBuilder reqBld = new RegistryRequestBuilder(); - String json = reqBld.createExportAllDataRequest("es_field_name", batchSize, searchAfter); - return json; + protected Search createRequest(Search req, int batchSize, String searchAfter) { + return req.all("es_field_name", batchSize, searchAfter); } } diff --git a/src/main/java/gov/nasa/pds/registry/mgr/srv/IndexService.java b/src/main/java/gov/nasa/pds/registry/mgr/srv/IndexService.java index 40c6458..860f7e7 100644 --- a/src/main/java/gov/nasa/pds/registry/mgr/srv/IndexService.java +++ b/src/main/java/gov/nasa/pds/registry/mgr/srv/IndexService.java @@ -8,7 +8,6 @@ import gov.nasa.pds.registry.common.RestClient; import gov.nasa.pds.registry.mgr.dao.IndexDao; import gov.nasa.pds.registry.mgr.dao.IndexSettings; -import gov.nasa.pds.registry.mgr.dao.RegistryRequestBuilder; /** * A service to work with Elasticsearch indices @@ -57,10 +56,8 @@ public void createIndex(String relativeSchemaPath, String indexName, int shards, log.info("Shards: " + shards); log.info("Replicas: " + replicas); - // Create request - RegistryRequestBuilder bld = new RegistryRequestBuilder(); - // Execute request - this.client.create(indexName, bld.createCreateIndexRequest(schemaFile, shards, replicas)); + // Execute request + this.client.create(indexName, RestClient.createCreateIndexRequest(schemaFile, shards, replicas)); } catch(ResponseException ex) { diff --git a/src/main/java/gov/nasa/pds/registry/mgr/util/es/EsQueryUtils.java b/src/main/java/gov/nasa/pds/registry/mgr/util/es/EsQueryUtils.java deleted file mode 100644 index 726a1ef..0000000 --- a/src/main/java/gov/nasa/pds/registry/mgr/util/es/EsQueryUtils.java +++ /dev/null @@ -1,97 +0,0 @@ -package gov.nasa.pds.registry.mgr.util.es; - -import java.io.IOException; - -import com.google.gson.stream.JsonWriter; - -/** - * Helper methods for building Elasticsearch queries. - * - * @author karpenko - */ -public class EsQueryUtils -{ - /** - * Append "match_all" object. - * @param writer JSON writer - * @throws IOException an exception - */ - public static void appendMatchAll(JsonWriter writer) throws IOException - { - writer.name("match_all"); - writer.beginObject(); - writer.endObject(); - } - - - /** - * Append match all query. - * @param writer JSON writer - * @throws IOException an exception - */ - public static void appendMatchAllQuery(JsonWriter writer) throws IOException - { - writer.name("query"); - writer.beginObject(); - appendMatchAll(writer); - writer.endObject(); - } - - - /** - * Append filter query - * @param writer JSON writer - * @param field field name - * @param value field value - * @throws IOException an exception - */ - public static void appendFilterQuery(JsonWriter writer, String field, String value) throws IOException - { - writer.name("query"); - writer.beginObject(); - - writer.name("bool"); - writer.beginObject(); - appendMustMatchAll(writer); - appendTermFilter(writer, field, value); - writer.endObject(); - - writer.endObject(); - } - - - /** - * Append must match all criterion - * @param writer JSON writer - * @throws IOException an exception - */ - private static void appendMustMatchAll(JsonWriter writer) throws IOException - { - writer.name("must"); - writer.beginObject(); - appendMatchAll(writer); - writer.endObject(); - } - - - /** - * Append term filter - * @param writer JSON writer - * @param field field name - * @param value field value - * @throws IOException an exception - */ - private static void appendTermFilter(JsonWriter writer, String field, String value) throws IOException - { - writer.name("filter"); - writer.beginObject(); - - writer.name("term"); - writer.beginObject(); - writer.name(field).value(value); - writer.endObject(); - - writer.endObject(); - } - -} diff --git a/src/test/java/tools/LoadLdds.java b/src/test/java/tools/LoadLdds.java index 565c54b..43b601b 100644 --- a/src/test/java/tools/LoadLdds.java +++ b/src/test/java/tools/LoadLdds.java @@ -4,8 +4,6 @@ import java.io.File; import java.io.FileReader; import gov.nasa.pds.registry.common.EstablishConnectionFactory; -import gov.nasa.pds.registry.common.cfg.RegistryCfg; -import gov.nasa.pds.registry.common.connection.AuthContent; import gov.nasa.pds.registry.common.es.dao.dd.DataDictionaryDao; import gov.nasa.pds.registry.common.es.service.JsonLddLoader; import gov.nasa.pds.registry.common.util.CloseUtils; @@ -17,15 +15,14 @@ public class LoadLdds public static void main(String[] args) throws Exception { - RegistryCfg cfg = new RegistryCfg(); - cfg.url = "app:/connections/direct/localhost.xml"; - cfg.indexName = "registry"; + String url = "app:/connections/direct/localhost.xml"; + String indexName = "registry"; BufferedReader rd = null; try { - RegistryManager.init(cfg); + RegistryManager.init(url, null, indexName); DataDictionaryDao ddDao = RegistryManager.getInstance().getDataDictionaryDao(); JsonLddLoader loader = new JsonLddLoader(ddDao, EstablishConnectionFactory.from("app:/connections/direct/localhost.xml").setIndexName("registry")); diff --git a/src/test/java/tt/TestDDRequestBuilder.java b/src/test/java/tt/TestDDRequestBuilder.java deleted file mode 100644 index 3f8e078..0000000 --- a/src/test/java/tt/TestDDRequestBuilder.java +++ /dev/null @@ -1,19 +0,0 @@ -package tt; - -import gov.nasa.pds.registry.common.es.dao.dd.DDRequestBuilder; - -public class TestDDRequestBuilder -{ - public static void main(String[] args) throws Exception - { - testGetLddInfoRequest(); - } - - - public static void testGetLddInfoRequest() throws Exception - { - DDRequestBuilder bld = new DDRequestBuilder(true); - String req = bld.createListLddsRequest("pds"); - System.out.println(req); - } -} diff --git a/src/test/java/tt/TestEsQueryBuilder.java b/src/test/java/tt/TestEsQueryBuilder.java deleted file mode 100644 index 7a5a8c1..0000000 --- a/src/test/java/tt/TestEsQueryBuilder.java +++ /dev/null @@ -1,33 +0,0 @@ -package tt; - -import gov.nasa.pds.registry.mgr.dao.RegistryRequestBuilder; - -public class TestEsQueryBuilder -{ - public static void main(String[] args) throws Exception - { - testMatchAll(); - System.out.println(); - - testDelete(); - System.out.println(); - } - - - private static void testMatchAll() throws Exception - { - RegistryRequestBuilder bld = new RegistryRequestBuilder(true); - String json = bld.createMatchAllQuery(); - System.out.println(json); - } - - - private static void testDelete() throws Exception - { - RegistryRequestBuilder bld = new RegistryRequestBuilder(true); - String json = bld.createFilterQuery("lidvid", "test::1.0"); - System.out.println(json); - } - - -} diff --git a/src/test/java/tt/TestLddLoader.java b/src/test/java/tt/TestLddLoader.java index 1cd4b13..0c3b16a 100644 --- a/src/test/java/tt/TestLddLoader.java +++ b/src/test/java/tt/TestLddLoader.java @@ -2,7 +2,6 @@ import java.io.File; import gov.nasa.pds.registry.common.EstablishConnectionFactory; -import gov.nasa.pds.registry.common.cfg.RegistryCfg; import gov.nasa.pds.registry.common.es.dao.dd.DataDictionaryDao; import gov.nasa.pds.registry.common.es.service.JsonLddLoader; import gov.nasa.pds.registry.mgr.dao.RegistryManager; @@ -13,13 +12,12 @@ public class TestLddLoader public static void main(String[] args) throws Exception { - RegistryCfg cfg = new RegistryCfg(); - cfg.url = "app:/connections/direct/localhost.xml"; - cfg.indexName = "registry"; + String url = "app:/connections/direct/localhost.xml"; + String indexName = "registry"; try { - RegistryManager.init(cfg); + RegistryManager.init(url, null, indexName); DataDictionaryDao ddDao = RegistryManager.getInstance().getDataDictionaryDao(); JsonLddLoader loader = new JsonLddLoader(ddDao, EstablishConnectionFactory.from("app:/connections/direct/localhost.xml").setIndexName("t1")); diff --git a/src/test/java/tt/TestRegistryRequestBld.java b/src/test/java/tt/TestRegistryRequestBld.java deleted file mode 100644 index 78f57bc..0000000 --- a/src/test/java/tt/TestRegistryRequestBld.java +++ /dev/null @@ -1,46 +0,0 @@ -package tt; - -import java.io.File; - -import gov.nasa.pds.registry.mgr.dao.RegistryRequestBuilder; - -public class TestRegistryRequestBld -{ - - public static void main(String[] args) throws Exception - { - - } - - - private static void testCreateRegistryRequest() throws Exception - { - RegistryRequestBuilder bld = new RegistryRequestBuilder(true); - - File schemaFile = new File("/tmp/schema/t2.json"); - String json = bld.createCreateIndexRequest(schemaFile, 3, 1); - System.out.println(json); - } - - - private static void testExportDataRequest() throws Exception - { - RegistryRequestBuilder bld = new RegistryRequestBuilder(true); - - String json = bld.createExportDataRequest("lidvid", "abc123", "lidvid", 100, null); - System.out.println(json); - - System.out.println(); - json = bld.createExportDataRequest("lidvid", "abc123", "lidvid", 100, "after::123::abc"); - System.out.println(json); - - System.out.println(); - json = bld.createExportAllDataRequest("lidvid", 100, null); - System.out.println(json); - - System.out.println(); - json = bld.createExportAllDataRequest("lidvid", 100, "after::123::abc"); - System.out.println(json); - } - -}