Skip to content

Commit

Permalink
clean up bad references
Browse files Browse the repository at this point in the history
  • Loading branch information
Al Niessner authored and Al Niessner committed Apr 29, 2024
1 parent 8d2667c commit 4ddb728
Show file tree
Hide file tree
Showing 21 changed files with 209 additions and 936 deletions.
2 changes: 1 addition & 1 deletion pom.xml
View file
Expand Up @@ -78,7 +78,7 @@ POSSIBILITY OF SUCH DAMAGE.
<dependency>
<groupId>gov.nasa.pds</groupId>
<artifactId>registry-common</artifactId>
<version>1.6.0</version>
<version>1.6.0-SNAPSHOT</version>
</dependency>
<!-- JSON parser -->
<dependency>
Expand Down
105 changes: 26 additions & 79 deletions src/main/java/gov/nasa/pds/registry/mgr/cmd/data/DeleteDataCmd.java
View file
@@ -1,24 +1,14 @@
package gov.nasa.pds.registry.mgr.cmd.data;

import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.Reader;
import java.util.Map;

import org.apache.commons.cli.CommandLine;
import org.elasticsearch.client.Request;
import org.elasticsearch.client.Response;
import org.elasticsearch.client.ResponseException;
import org.elasticsearch.client.RestClient;

import com.google.gson.Gson;

import gov.nasa.pds.registry.common.es.client.EsClientFactory;
import gov.nasa.pds.registry.common.es.client.EsUtils;
import gov.nasa.pds.registry.common.util.CloseUtils;
import gov.nasa.pds.registry.common.ConnectionFactory;
import gov.nasa.pds.registry.common.EstablishConnectionFactory;
import gov.nasa.pds.registry.common.Request;
import gov.nasa.pds.registry.common.ResponseException;
import gov.nasa.pds.registry.common.RestClient;
import gov.nasa.pds.registry.mgr.Constants;
import gov.nasa.pds.registry.mgr.cmd.CliCommand;
import gov.nasa.pds.registry.mgr.dao.RegistryRequestBuilder;


/**
Expand All @@ -30,8 +20,6 @@
public class DeleteDataCmd implements CliCommand
{
private String filterMessage;
private String regQuery;
private String refsQuery;


/**
Expand All @@ -55,116 +43,75 @@ public void run(CommandLine cmdLine) throws Exception
String indexName = cmdLine.getOptionValue("index", Constants.DEFAULT_REGISTRY_INDEX);
String authPath = cmdLine.getOptionValue("auth");

buildEsQuery(cmdLine);

System.out.println("Elasticsearch URL: " + esUrl);
System.out.println(" Index: " + indexName);
System.out.println(filterMessage);
System.out.println();

RestClient client = null;

try
{
client = EsClientFactory.createRestClient(esUrl, authPath);
ConnectionFactory conFact = EstablishConnectionFactory.from(esUrl, authPath);
try (RestClient client = conFact.createRestClient()) {
Request.DeleteByQuery regQuery = client.createDeleteByQuery().setIndex(indexName),
refQuery = client.createDeleteByQuery().setIndex(indexName + "-refs");
buildEsQuery(cmdLine, regQuery, refQuery);
// Delete from registry index
deleteByQuery(client, indexName, regQuery);
deleteByQuery(indexName, client.performRequest(regQuery));
// Delete from product references index
deleteByQuery(client, indexName + "-refs", refsQuery);
deleteByQuery(indexName + "-refs", client.performRequest(refQuery));
}
catch(ResponseException ex)
{
throw new Exception(EsUtils.extractErrorMessage(ex));
}
finally
{
CloseUtils.close(client);
throw new Exception(ex.extractErrorMessage());
}
}


private static void deleteByQuery(RestClient client, String indexName, String query) throws Exception
{
Request req = new Request("POST", "/" + indexName + "/_delete_by_query");
req.setJsonEntity(query);

Response resp = client.performRequest(req);
double numDeleted = extractNumDeleted(resp);

System.out.format("Deleted %.0f document(s) from %s index\n", numDeleted, indexName);
}


@SuppressWarnings("rawtypes")
private static double extractNumDeleted(Response resp)
private static void deleteByQuery(String indexName, long numDeleted) throws Exception
{
try
{
InputStream is = resp.getEntity().getContent();
Reader rd = new InputStreamReader(is);

Gson gson = new Gson();
Object obj = gson.fromJson(rd, Object.class);
rd.close();

obj = ((Map)obj).get("deleted");
return (Double)obj;
}
catch(Exception ex)
{
return 0;
}
System.out.format("Deleted %d document(s) from %s index\n", numDeleted, indexName);
}


/**
* Build Elasticsearch query to delete records.
* Records can be deleted by LIDVID, LID, PackageID. All records can also be deleted.
* @param cmdLine
* @throws Exception
*/
private void buildEsQuery(CommandLine cmdLine) throws Exception
{
// Registry index
RegistryRequestBuilder regBld = new RegistryRequestBuilder();
// Product references index
RegistryRequestBuilder refsBld = new RegistryRequestBuilder();

private void buildEsQuery(CommandLine cmdLine, Request.DeleteByQuery regQuery, Request.DeleteByQuery refsQuery) throws Exception
{
String id = cmdLine.getOptionValue("lidvid");
if(id != null)
{
this.filterMessage = " LIDVID: " + id;
this.regQuery = regBld.createFilterQuery("lidvid", id);
this.refsQuery = refsBld.createFilterQuery("collection_lidvid", id);

regQuery.createFilterQuery("lidvid", id);
refsQuery.createFilterQuery("collection_lidvid", id);
return;
}

id = cmdLine.getOptionValue("lid");
if(id != null)
{
this.filterMessage = " LID: " + id;
this.regQuery = regBld.createFilterQuery("lid", id);
this.refsQuery = refsBld.createFilterQuery("collection_lid", id);

regQuery.createFilterQuery("lid", id);
refsQuery.createFilterQuery("collection_lid", id);
return;
}

id = cmdLine.getOptionValue("packageId");
if(id != null)
{
this.filterMessage = " Package ID: " + id;
this.regQuery = regBld.createFilterQuery("_package_id", id);
this.refsQuery = refsBld.createFilterQuery("_package_id", id);
regQuery.createFilterQuery("_package_id", id);
refsQuery.createFilterQuery("_package_id", id);

return;
}

if(cmdLine.hasOption("all"))
{
this.filterMessage = "Delete all documents ";
this.regQuery = regBld.createMatchAllQuery();
this.refsQuery = refsBld.createMatchAllQuery();
regQuery.createMatchAllQuery();
refsQuery.createMatchAllQuery();

return;
}
Expand Down
105 changes: 21 additions & 84 deletions src/main/java/gov/nasa/pds/registry/mgr/cmd/data/ExportFileCmd.java
View file
@@ -1,20 +1,13 @@
package gov.nasa.pds.registry.mgr.cmd.data;

import java.util.Map;

import org.apache.commons.cli.CommandLine;
import org.elasticsearch.client.Request;
import org.elasticsearch.client.Response;
import org.elasticsearch.client.ResponseException;
import org.elasticsearch.client.RestClient;

import gov.nasa.pds.registry.common.es.client.EsClientFactory;
import gov.nasa.pds.registry.common.es.client.EsUtils;
import gov.nasa.pds.registry.common.es.client.SearchResponseParser;
import gov.nasa.pds.registry.common.util.CloseUtils;
import gov.nasa.pds.registry.common.ConnectionFactory;
import gov.nasa.pds.registry.common.EstablishConnectionFactory;
import gov.nasa.pds.registry.common.Request;
import gov.nasa.pds.registry.common.ResponseException;
import gov.nasa.pds.registry.common.RestClient;
import gov.nasa.pds.registry.mgr.Constants;
import gov.nasa.pds.registry.mgr.cmd.CliCommand;
import gov.nasa.pds.registry.mgr.dao.RegistryRequestBuilder;
import gov.nasa.pds.registry.mgr.util.EmbeddedBlobExporter;


Expand All @@ -30,50 +23,6 @@ public class ExportFileCmd implements CliCommand
*
* @author karpenko
*/
private static class ResponseCB implements SearchResponseParser.Callback
{
private boolean found = false;
private String lidvid;
private String filePath;

/**
* Constructor
* @param lidvid LidVid of a document with BLOB
* @param filePath File path to export BLOB to.
*/
public ResponseCB(String lidvid, String filePath)
{
this.lidvid = lidvid;
this.filePath = filePath;
}


@Override
@SuppressWarnings("rawtypes")
public void onRecord(String id, Object rec) throws Exception
{
found = true;

Object blob = ((Map)rec).get(Constants.BLOB_FIELD);
if(blob == null)
{
System.out.println("There is no BLOB in a document with LIDVID = " + lidvid);
System.out.println("Probably embedded BLOB storage was not enabled when the document was created.");
return;
}

EmbeddedBlobExporter.export(blob.toString(), filePath);
System.out.println("Done");
}


public boolean found()
{
return found;
}

}


/**
* Constructor
Expand Down Expand Up @@ -116,39 +65,27 @@ public void run(CommandLine cmdLine) throws Exception
System.out.println(" Output file: " + filePath);
System.out.println();

RestClient client = null;

try
ConnectionFactory conFact = EstablishConnectionFactory.from(esUrl, authPath);
try (RestClient client = conFact.createRestClient())
{
// Create Elasticsearch client
client = EsClientFactory.createRestClient(esUrl, authPath);

// Create request
Request req = new Request("GET", "/" + indexName + "/_search");
RegistryRequestBuilder bld = new RegistryRequestBuilder();
String jsonReq = bld.createGetBlobRequest(lidvid);
req.setJsonEntity(jsonReq);

// Execute request
Response resp = client.performRequest(req);
Request.Search req = client.createSearchRequest().setIndex(indexName).buildGetField(Constants.BLOB_FIELD, lidvid);
String blob = client.performRequest(req).field(Constants.BLOB_FIELD);
if(blob == null)
{
System.out.println("There is no BLOB in a document with LIDVID = " + lidvid);
System.out.println("Probably embedded BLOB storage was not enabled when the document was created.");
return;
}

SearchResponseParser respParser = new SearchResponseParser();
ResponseCB cb = new ResponseCB(lidvid, filePath);
respParser.parseResponse(resp, cb);

if(!cb.found())
{
System.out.println("Could not find a document with lidvid = " + lidvid);
return;
}
EmbeddedBlobExporter.export(blob, filePath);
System.out.println("Done");
}
catch (NoSuchFieldException e) {
System.out.println("No documents found matching lidvid: " + lidvid);
}
catch(ResponseException ex)
{
throw new Exception(EsUtils.extractErrorMessage(ex));
}
finally
{
CloseUtils.close(client);
throw new Exception(ex.extractErrorMessage());
}
}

Expand Down
18 changes: 6 additions & 12 deletions src/main/java/gov/nasa/pds/registry/mgr/cmd/data/UpdateAltIdsCmd.java
View file
Expand Up @@ -14,10 +14,7 @@
import org.apache.commons.lang3.StringUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.elasticsearch.client.ResponseException;

import gov.nasa.pds.registry.common.cfg.RegistryCfg;
import gov.nasa.pds.registry.common.es.client.EsUtils;
import gov.nasa.pds.registry.common.ResponseException;
import gov.nasa.pds.registry.common.util.CloseUtils;
import gov.nasa.pds.registry.mgr.Constants;
import gov.nasa.pds.registry.mgr.cmd.CliCommand;
Expand Down Expand Up @@ -51,25 +48,22 @@ public void run(CommandLine cmdLine) throws Exception
printHelp();
return;
}

RegistryCfg cfg = new RegistryCfg();
cfg.url = cmdLine.getOptionValue("es", "app:/connections/direct/localhost.xml");
cfg.indexName = cmdLine.getOptionValue("index", Constants.DEFAULT_REGISTRY_INDEX);
cfg.authFile = cmdLine.getOptionValue("auth");

String url = cmdLine.getOptionValue("es", "app:/connections/direct/localhost.xml");
String indexName = cmdLine.getOptionValue("index", Constants.DEFAULT_REGISTRY_INDEX);
String authFile = cmdLine.getOptionValue("auth");
String filePath = cmdLine.getOptionValue("file");
if(filePath == null) throw new Exception("Missing required parameter '-file'");
File file = new File(filePath);
if(!file.exists()) throw new Exception("Input file doesn't exist: " + file.getAbsolutePath());

try
{
RegistryManager.init(cfg);
RegistryManager.init(url, authFile, indexName);
updateIds(file);
}
catch(ResponseException ex)
{
throw new Exception(EsUtils.extractErrorMessage(ex));
throw new Exception(ex.extractErrorMessage());
}
finally
{
Expand Down

0 comments on commit 4ddb728

Please sign in to comment.