diff --git a/caom2-meta-sync/Dockerfile b/caom2-meta-sync/Dockerfile deleted file mode 100644 index 16942dd7..00000000 --- a/caom2-meta-sync/Dockerfile +++ /dev/null @@ -1,5 +0,0 @@ -FROM images.opencadc.org/library/cadc-java:1 - -ADD build/distributions/caom2-meta-sync.tar / - -CMD ["/caom2-meta-sync/bin/caom2-meta-sync"] diff --git a/darkspire/Dockerfile b/darkspire/Dockerfile new file mode 100644 index 00000000..dd1ff33b --- /dev/null +++ b/darkspire/Dockerfile @@ -0,0 +1,5 @@ +FROM images.opencadc.org/library/cadc-java:1 + +ADD build/distributions/darkspire.tar / + +CMD ["/darkspire/bin/darkspire"] diff --git a/caom2-meta-sync/README.md b/darkspire/README.md similarity index 100% rename from caom2-meta-sync/README.md rename to darkspire/README.md diff --git a/caom2-meta-sync/VERSION b/darkspire/VERSION similarity index 100% rename from caom2-meta-sync/VERSION rename to darkspire/VERSION diff --git a/caom2-meta-sync/build.gradle b/darkspire/build.gradle similarity index 62% rename from caom2-meta-sync/build.gradle rename to darkspire/build.gradle index 1c5cc802..4f1b7094 100644 --- a/caom2-meta-sync/build.gradle +++ b/darkspire/build.gradle @@ -14,10 +14,7 @@ sourceCompatibility = 1.8 group = 'org.opencadc' -description = 'OpenCADC CAOM Metadata Sync application' -def git_url = 'https://github.com/opencadc/caom2db' - -mainClassName = 'org.opencadc.caom2.metasync.Main' +mainClassName = 'org.opencadc.darkspire.Main' dependencies { implementation 'org.opencadc:cadc-util:[1.6,2.0)' @@ -25,12 +22,9 @@ dependencies { implementation 'org.opencadc:caom2persistence:[2.4.14,2.5)' implementation 'org.opencadc:caom2-repo:[1.4,1.5)' - // needed to run plane metadata compute plugin (--compute) + // needed for validation implementation 'org.opencadc:caom2-compute:[2.4.6,2.5)' - // needed to run access-control regen plugin (--generate-ac) - implementation 'org.opencadc:caom2-access-control:[2.4,2.5)' - runtimeOnly 'org.postgresql:postgresql:[42.2,43.0)' } diff --git a/caom2-meta-sync/src/main/java/org/opencadc/caom2/metasync/CaomHarvester.java b/darkspire/src/main/java/org/opencadc/darkspire/CaomHarvester.java similarity index 91% rename from caom2-meta-sync/src/main/java/org/opencadc/caom2/metasync/CaomHarvester.java rename to darkspire/src/main/java/org/opencadc/darkspire/CaomHarvester.java index 556206af..5186abc9 100644 --- a/caom2-meta-sync/src/main/java/org/opencadc/caom2/metasync/CaomHarvester.java +++ b/darkspire/src/main/java/org/opencadc/darkspire/CaomHarvester.java @@ -67,7 +67,7 @@ ************************************************************************ */ -package org.opencadc.caom2.metasync; +package org.opencadc.darkspire; import ca.nrc.cadc.caom2.DeletedObservation; import ca.nrc.cadc.caom2.harvester.state.HarvestState; @@ -102,8 +102,6 @@ public class CaomHarvester implements Runnable { private final boolean nochecksum; private final boolean exitWhenComplete; private final long maxIdle; - private boolean computePlaneMetadata; - private File readAccessConfigFile; /** * Harvest everything. @@ -134,8 +132,6 @@ public CaomHarvester(HarvesterResource src, HarvesterResource dest, List this.nochecksum = nochecksum; this.exitWhenComplete = exitWhenComplete; this.maxIdle = maxIdle; - this.computePlaneMetadata = false; - this.readAccessConfigFile = null; ConnectionConfig cc = new ConnectionConfig(null, null, dest.getUsername(), dest.getPassword(), HarvesterResource.POSTGRESQL_DRIVER, dest.getJdbcUrl()); @@ -143,24 +139,6 @@ public CaomHarvester(HarvesterResource src, HarvesterResource dest, List this.initdb = new InitDatabase(ds, dest.getDatabase(), dest.getSchema()); } - /** - * Enable the plane metadata compute plugin. - * - * @param compute enable Plane metadata computation if true - */ - public void setCompute(boolean compute) { - this.computePlaneMetadata = compute; - } - - /** - * Enable the generate read access grants plugin with the specified config. - * - * @param config enable read access generation from the specified config file - */ - public void setGenerateReadAccess(String config) { - this.readAccessConfigFile = new File(config); - } - @Override public void run() { @@ -199,10 +177,6 @@ public void run() { ObservationHarvester obsHarvester = new ObservationHarvester(src, dest, collection, basePublisherID, batchSize, nthreads, full, nochecksum); obsHarvester.setSkipped(skip); - obsHarvester.setComputePlaneMetadata(computePlaneMetadata); - if (readAccessConfigFile != null) { - obsHarvester.setGenerateReadAccessTuples(readAccessConfigFile); - } DeletionHarvester obsDeleter = new DeletionHarvester(DeletedObservation.class, src, dest, collection, batchSize * 100); diff --git a/caom2-meta-sync/src/main/java/org/opencadc/caom2/metasync/DeletionHarvester.java b/darkspire/src/main/java/org/opencadc/darkspire/DeletionHarvester.java similarity index 92% rename from caom2-meta-sync/src/main/java/org/opencadc/caom2/metasync/DeletionHarvester.java rename to darkspire/src/main/java/org/opencadc/darkspire/DeletionHarvester.java index b6ea5e82..a68dc156 100644 --- a/caom2-meta-sync/src/main/java/org/opencadc/caom2/metasync/DeletionHarvester.java +++ b/darkspire/src/main/java/org/opencadc/darkspire/DeletionHarvester.java @@ -67,7 +67,7 @@ ************************************************************************ */ -package org.opencadc.caom2.metasync; +package org.opencadc.darkspire; import ca.nrc.cadc.caom2.DeletedEntity; import ca.nrc.cadc.caom2.DeletedObservation; @@ -136,24 +136,7 @@ public void setInitHarvestState(boolean initHarvestState) { */ private void init() { // source - if (src.getResourceType() == HarvesterResource.SOURCE_DB) { - Map srcConfig = getConfigDAO(src); - ConnectionConfig srcConnectionConfig = new ConnectionConfig(null, null, - src.getUsername(), src.getPassword(), HarvesterResource.POSTGRESQL_DRIVER, src.getJdbcUrl()); - final String srcDS = "jdbc/srcDelHarvest"; - try { - DBUtil.createJNDIDataSource(srcDS, srcConnectionConfig); - } catch (NamingException e) { - throw new IllegalStateException(String.format("Error creating source JNDI datasource for %s reason: %s", - src, e.getMessage())); - } - srcConfig.put("jndiDataSourceName", srcDS); - this.deletedDAO = new DeletedEntityDAO(); - deletedDAO.setConfig(srcConfig); - ready = true; - } else { - this.repoClient = new RepoClient(src.getResourceID(), 1); - } + this.repoClient = new RepoClient(src.getResourceID(), 1); // destination Map destConfig = getConfigDAO(dest); @@ -172,12 +155,10 @@ private void init() { this.txnManager = obsDAO.getTransactionManager(); initHarvestState(obsDAO.getDataSource(), entityClass); - if (repoClient != null) { - if (repoClient.isDelAvailable()) { - ready = true; - } else { - log.warn("Not available deletion endpoint in " + repoClient.toString()); - } + if (repoClient.isDelAvailable()) { + ready = true; + } else { + log.warn("Not available deletion endpoint in " + repoClient.toString()); } } diff --git a/caom2-meta-sync/src/main/java/org/opencadc/caom2/metasync/Harvester.java b/darkspire/src/main/java/org/opencadc/darkspire/Harvester.java similarity index 99% rename from caom2-meta-sync/src/main/java/org/opencadc/caom2/metasync/Harvester.java rename to darkspire/src/main/java/org/opencadc/darkspire/Harvester.java index 88c0570a..21ec9196 100644 --- a/caom2-meta-sync/src/main/java/org/opencadc/caom2/metasync/Harvester.java +++ b/darkspire/src/main/java/org/opencadc/darkspire/Harvester.java @@ -67,7 +67,7 @@ ************************************************************************ */ -package org.opencadc.caom2.metasync; +package org.opencadc.darkspire; import ca.nrc.cadc.caom2.harvester.state.HarvestStateDAO; import ca.nrc.cadc.caom2.harvester.state.PostgresqlHarvestStateDAO; diff --git a/caom2-meta-sync/src/main/java/org/opencadc/caom2/metasync/HarvesterResource.java b/darkspire/src/main/java/org/opencadc/darkspire/HarvesterResource.java similarity index 99% rename from caom2-meta-sync/src/main/java/org/opencadc/caom2/metasync/HarvesterResource.java rename to darkspire/src/main/java/org/opencadc/darkspire/HarvesterResource.java index d37bd97b..9f614677 100644 --- a/caom2-meta-sync/src/main/java/org/opencadc/caom2/metasync/HarvesterResource.java +++ b/darkspire/src/main/java/org/opencadc/darkspire/HarvesterResource.java @@ -65,7 +65,7 @@ ************************************************************************ */ -package org.opencadc.caom2.metasync; +package org.opencadc.darkspire; import java.net.URI; import org.apache.log4j.Logger; diff --git a/caom2-meta-sync/src/main/java/org/opencadc/caom2/metasync/Main.java b/darkspire/src/main/java/org/opencadc/darkspire/Main.java similarity index 99% rename from caom2-meta-sync/src/main/java/org/opencadc/caom2/metasync/Main.java rename to darkspire/src/main/java/org/opencadc/darkspire/Main.java index 8e2b7896..bd7c0e4c 100644 --- a/caom2-meta-sync/src/main/java/org/opencadc/caom2/metasync/Main.java +++ b/darkspire/src/main/java/org/opencadc/darkspire/Main.java @@ -67,7 +67,7 @@ ************************************************************************ */ -package org.opencadc.caom2.metasync; +package org.opencadc.darkspire; import ca.nrc.cadc.auth.RunnableAction; import ca.nrc.cadc.auth.SSLUtil; @@ -92,7 +92,7 @@ public class Main { private static final Logger log = Logger.getLogger(Main.class); - private static final String CONFIG_FILE_NAME = "caom2-meta-sync.properties"; + private static final String CONFIG_FILE_NAME = "darkspire.properties"; private static final String CERTIFICATE_FILE_LOCATION = System.getProperty("user.home") + "/.ssl/cadcproxy.pem"; private static final String CONFIG_PREFIX = Main.class.getPackage().getName(); private static final String LOGGING_CONFIG_KEY = CONFIG_PREFIX + ".logging"; diff --git a/caom2-meta-sync/src/main/java/org/opencadc/caom2/metasync/MismatchedChecksumException.java b/darkspire/src/main/java/org/opencadc/darkspire/MismatchedChecksumException.java similarity index 99% rename from caom2-meta-sync/src/main/java/org/opencadc/caom2/metasync/MismatchedChecksumException.java rename to darkspire/src/main/java/org/opencadc/darkspire/MismatchedChecksumException.java index ec764fa8..95b644ae 100644 --- a/caom2-meta-sync/src/main/java/org/opencadc/caom2/metasync/MismatchedChecksumException.java +++ b/darkspire/src/main/java/org/opencadc/darkspire/MismatchedChecksumException.java @@ -67,7 +67,7 @@ ************************************************************************ */ -package org.opencadc.caom2.metasync; +package org.opencadc.darkspire; public class MismatchedChecksumException extends Exception { diff --git a/caom2-meta-sync/src/main/java/org/opencadc/caom2/metasync/ObservationHarvester.java b/darkspire/src/main/java/org/opencadc/darkspire/ObservationHarvester.java similarity index 89% rename from caom2-meta-sync/src/main/java/org/opencadc/caom2/metasync/ObservationHarvester.java rename to darkspire/src/main/java/org/opencadc/darkspire/ObservationHarvester.java index ed42771c..a4c40446 100644 --- a/caom2-meta-sync/src/main/java/org/opencadc/caom2/metasync/ObservationHarvester.java +++ b/darkspire/src/main/java/org/opencadc/darkspire/ObservationHarvester.java @@ -67,7 +67,7 @@ ************************************************************************ */ -package org.opencadc.caom2.metasync; +package org.opencadc.darkspire; import ca.nrc.cadc.caom2.Artifact; import ca.nrc.cadc.caom2.Observation; @@ -75,9 +75,7 @@ import ca.nrc.cadc.caom2.ObservationState; import ca.nrc.cadc.caom2.ObservationURI; import ca.nrc.cadc.caom2.Plane; -import ca.nrc.cadc.caom2.ac.ReadAccessGenerator; import ca.nrc.cadc.caom2.compute.CaomWCSValidator; -import ca.nrc.cadc.caom2.compute.ComputeUtil; import ca.nrc.cadc.caom2.harvester.state.HarvestSkipURI; import ca.nrc.cadc.caom2.harvester.state.HarvestSkipURIDAO; import ca.nrc.cadc.caom2.harvester.state.HarvestState; @@ -128,7 +126,6 @@ public class ObservationHarvester extends Harvester { private HarvestSkipURIDAO harvestSkipDAO; private boolean skipped; private boolean computePlaneMetadata; - private ReadAccessGenerator acGenerator; private boolean ready = false; private int ingested = 0; @@ -145,88 +142,14 @@ public void setSkipped(boolean skipped) { this.skipped = skipped; } - public boolean getComputePlaneMetadata() { - return this.computePlaneMetadata; - } - - public void setComputePlaneMetadata(boolean computePlaneMetadata) { - this.computePlaneMetadata = computePlaneMetadata; - } - - public void setGenerateReadAccessTuples(File config) { - try { - Properties props = new Properties(); - props.load(new FileReader(config)); - String line = props.getProperty(collection); - if (line == null) { - throw new IllegalArgumentException("CONFIG: collection not found: " + collection); - } - - Map groupConfig = new HashMap<>(); - String[] parts = line.split("[ \t]+"); // one or more spaces and tabs - for (int i = 0; i < parts.length; i++) { - String option = parts[i]; // key=value pair - String[] kv = option.split("="); - if (kv.length != 2) { - throw new IllegalArgumentException("invalid key=value pair: " + option); - } - if (ReadAccessGenerator.PROPOSAL_GROUP_KEY.equals(kv[0])) { - boolean proposalGroup = "true".equals(kv[1]); - if (proposalGroup) { - groupConfig.put(ReadAccessGenerator.PROPOSAL_GROUP_KEY, proposalGroup); - } - } else if (ReadAccessGenerator.OPERATOR_GROUP_KEY.equals(kv[0])) { - String og = kv[1]; - if (og != null) { - URI ouri = new URI(og); - groupConfig.put(ReadAccessGenerator.OPERATOR_GROUP_KEY, ouri); - } - } else if (ReadAccessGenerator.STAFF_GROUP_KEY.equals(kv[0])) { - String sg = kv[1]; - if (sg != null) { - URI suri = new URI(sg); - groupConfig.put(ReadAccessGenerator.STAFF_GROUP_KEY, suri); - } - } - } - for (Map.Entry me : groupConfig.entrySet()) { - log.debug("generate config for " + collection + ": " + me.getKey() + " = " + me.getValue()); - } - this.acGenerator = new ReadAccessGenerator(collection, groupConfig); - } catch (IOException ex) { - throw new RuntimeException("failed to read config from " + config, ex); - } catch (Exception ex) { - throw new RuntimeException("CONFIG: invalid config " + config, ex); - } - } - public int getIngested() { return this.ingested; } private void init(int nthreads) { - if (src.getResourceType() == HarvesterResource.SOURCE_DB) { - Map srcConfig = getConfigDAO(src); - ConnectionConfig srcConnectionConfig = new ConnectionConfig(null, null, - src.getUsername(), src.getPassword(), HarvesterResource.POSTGRESQL_DRIVER, src.getJdbcUrl()); - final String srcDS = "jdbc/obsHarvestSrc"; - try { - DBUtil.createJNDIDataSource(srcDS, srcConnectionConfig); - } catch (NamingException e) { - throw new IllegalStateException(String.format("Error creating source JNDI datasource for %s reason: %s", - src, e.getMessage())); - } - srcConfig.put("jndiDataSourceName", srcDS); - this.srcObservationDAO = new ObservationDAO(); - srcObservationDAO.setConfig(srcConfig); - ready = true; - } else if (src.getResourceType() == HarvesterResource.SOURCE_URI) { - this.srcObservationService = new RepoClient(src.getResourceID(), nthreads); - } else { - throw new IllegalStateException("BUG: unexpected HarvestResource resource type: " + src); - } + this.srcObservationService = new RepoClient(src.getResourceID(), nthreads); - // for now, dest is always a database + // dest is always a database Map destConfig = getConfigDAO(dest); ConnectionConfig destConnectionConfig = new ConnectionConfig(null, null, dest.getUsername(), dest.getPassword(), HarvesterResource.POSTGRESQL_DRIVER, dest.getJdbcUrl()); @@ -249,12 +172,10 @@ private void init(int nthreads) { } initHarvestState(destObservationDAO.getDataSource(), Observation.class); - if (srcObservationService != null) { - if (srcObservationService.isObsAvailable()) { - ready = true; - } else { - log.error("Not available obs endpoint in " + srcObservationService.toString()); - } + if (srcObservationService.isObsAvailable()) { + ready = true; + } else { + log.error("Not available: obs endpoint in " + srcObservationService.toString()); } } diff --git a/caom2-meta-sync/src/main/java/org/opencadc/caom2/metasync/SkippedWrapperURI.java b/darkspire/src/main/java/org/opencadc/darkspire/SkippedWrapperURI.java similarity index 99% rename from caom2-meta-sync/src/main/java/org/opencadc/caom2/metasync/SkippedWrapperURI.java rename to darkspire/src/main/java/org/opencadc/darkspire/SkippedWrapperURI.java index ee077159..4d7a089c 100644 --- a/caom2-meta-sync/src/main/java/org/opencadc/caom2/metasync/SkippedWrapperURI.java +++ b/darkspire/src/main/java/org/opencadc/darkspire/SkippedWrapperURI.java @@ -67,7 +67,7 @@ ************************************************************************ */ -package org.opencadc.caom2.metasync; +package org.opencadc.darkspire; import ca.nrc.cadc.caom2.harvester.state.HarvestSkipURI; import org.apache.log4j.Logger;