rename caom2-meta-sync to darkspire

opencadc · Jun 22, 2023 · 899a3df · 899a3df
1 parent f5b73a8
commit 899a3df
Show file tree

Hide file tree

Showing 13 changed files with 27 additions and 157 deletions.
diff --git a/caom2-meta-sync/Dockerfile b/caom2-meta-sync/Dockerfile
diff --git a/darkspire/Dockerfile b/darkspire/Dockerfile
@@ -0,0 +1,5 @@
+FROM images.opencadc.org/library/cadc-java:1
+
+ADD build/distributions/darkspire.tar /
+
+CMD ["/darkspire/bin/darkspire"]
diff --git a/caom2-meta-sync/README.md → darkspire/README.md b/caom2-meta-sync/README.md → darkspire/README.md
diff --git a/caom2-meta-sync/VERSION → darkspire/VERSION b/caom2-meta-sync/VERSION → darkspire/VERSION
diff --git a/caom2-meta-sync/build.gradle → darkspire/build.gradle b/caom2-meta-sync/build.gradle → darkspire/build.gradle
@@ -14,23 +14,17 @@ sourceCompatibility = 1.8
 
 group = 'org.opencadc'
 
-description = 'OpenCADC CAOM Metadata Sync application'
-def git_url = 'https://github.com/opencadc/caom2db'
-
-mainClassName = 'org.opencadc.caom2.metasync.Main'
+mainClassName = 'org.opencadc.darkspire.Main'
 
 dependencies {
     implementation 'org.opencadc:cadc-util:[1.6,2.0)'
     implementation 'org.opencadc:caom2:[2.4.4,2.5)'
     implementation 'org.opencadc:caom2persistence:[2.4.14,2.5)'
     implementation 'org.opencadc:caom2-repo:[1.4,1.5)'
 
-    // needed to run plane metadata compute plugin (--compute)
+    // needed for validation
     implementation 'org.opencadc:caom2-compute:[2.4.6,2.5)'
 
-    // needed to run access-control regen plugin (--generate-ac)
-    implementation 'org.opencadc:caom2-access-control:[2.4,2.5)'
-
     runtimeOnly 'org.postgresql:postgresql:[42.2,43.0)'
 }
 

diff --git a/...pencadc/caom2/metasync/CaomHarvester.java → ...org/opencadc/darkspire/CaomHarvester.java b/...pencadc/caom2/metasync/CaomHarvester.java → ...org/opencadc/darkspire/CaomHarvester.java
@@ -67,7 +67,7 @@
  ************************************************************************
  */
 
-package org.opencadc.caom2.metasync;
+package org.opencadc.darkspire;
 
 import ca.nrc.cadc.caom2.DeletedObservation;
 import ca.nrc.cadc.caom2.harvester.state.HarvestState;
@@ -102,8 +102,6 @@ public class CaomHarvester implements Runnable {
     private final boolean nochecksum;
     private final boolean exitWhenComplete;
     private final long maxIdle;
-    private boolean computePlaneMetadata;
-    private File readAccessConfigFile;
 
     /**
      * Harvest everything.
@@ -134,33 +132,13 @@ public CaomHarvester(HarvesterResource src, HarvesterResource dest, List<String>
         this.nochecksum = nochecksum;
         this.exitWhenComplete = exitWhenComplete;
         this.maxIdle = maxIdle;
-        this.computePlaneMetadata = false;
-        this.readAccessConfigFile = null;
 
         ConnectionConfig cc = new ConnectionConfig(null, null, dest.getUsername(), dest.getPassword(),
                 HarvesterResource.POSTGRESQL_DRIVER, dest.getJdbcUrl());
         DataSource ds = DBUtil.getDataSource(cc);
         this.initdb = new InitDatabase(ds, dest.getDatabase(), dest.getSchema());
     }
 
-    /**
-     * Enable the plane metadata compute plugin.
-     * 
-     * @param compute enable Plane metadata computation if true
-     */
-    public void setCompute(boolean compute) {
-        this.computePlaneMetadata = compute;
-    }
-
-    /**
-     * Enable the generate read access grants plugin with the specified config.
-     * 
-     * @param config enable read access generation from the specified config file
-     */
-    public void setGenerateReadAccess(String config) {
-        this.readAccessConfigFile = new File(config);
-    }
-
     @Override
     public void run() {
 
@@ -199,10 +177,6 @@ public void run() {
                 ObservationHarvester obsHarvester = new ObservationHarvester(src, dest, collection, basePublisherID, batchSize,
                         nthreads, full, nochecksum);
                 obsHarvester.setSkipped(skip);
-                obsHarvester.setComputePlaneMetadata(computePlaneMetadata);
-                if (readAccessConfigFile != null) {
-                    obsHarvester.setGenerateReadAccessTuples(readAccessConfigFile);
-                }
 
                 DeletionHarvester obsDeleter = new DeletionHarvester(DeletedObservation.class, src, dest,
                         collection, batchSize * 100);

diff --git a/...adc/caom2/metasync/DeletionHarvester.java → ...opencadc/darkspire/DeletionHarvester.java b/...adc/caom2/metasync/DeletionHarvester.java → ...opencadc/darkspire/DeletionHarvester.java
@@ -67,7 +67,7 @@
  ************************************************************************
  */
 
-package org.opencadc.caom2.metasync;
+package org.opencadc.darkspire;
 
 import ca.nrc.cadc.caom2.DeletedEntity;
 import ca.nrc.cadc.caom2.DeletedObservation;
@@ -136,24 +136,7 @@ public void setInitHarvestState(boolean initHarvestState) {
      */
     private void init() {
         // source
-        if (src.getResourceType() == HarvesterResource.SOURCE_DB) {
-            Map<String, Object> srcConfig = getConfigDAO(src);
-            ConnectionConfig srcConnectionConfig = new ConnectionConfig(null, null,
-                    src.getUsername(), src.getPassword(), HarvesterResource.POSTGRESQL_DRIVER, src.getJdbcUrl());
-            final String srcDS = "jdbc/srcDelHarvest";
-            try {
-                DBUtil.createJNDIDataSource(srcDS, srcConnectionConfig);
-            } catch (NamingException e) {
-                throw new IllegalStateException(String.format("Error creating source JNDI datasource for %s reason: %s",
-                        src, e.getMessage()));
-            }
-            srcConfig.put("jndiDataSourceName", srcDS);
-            this.deletedDAO = new DeletedEntityDAO();
-            deletedDAO.setConfig(srcConfig);
-            ready = true;
-        } else {
-            this.repoClient = new RepoClient(src.getResourceID(), 1);
-        }
+        this.repoClient = new RepoClient(src.getResourceID(), 1);
 
         // destination
         Map<String, Object> destConfig = getConfigDAO(dest);
@@ -172,12 +155,10 @@ private void init() {
         this.txnManager = obsDAO.getTransactionManager();
         initHarvestState(obsDAO.getDataSource(), entityClass);
 
-        if (repoClient != null) {
-            if (repoClient.isDelAvailable()) {
-                ready = true;
-            } else {
-                log.warn("Not available deletion endpoint in " + repoClient.toString());
-            }
+        if (repoClient.isDelAvailable()) {
+            ready = true;
+        } else {
+            log.warn("Not available deletion endpoint in " + repoClient.toString());
         }
     }
 

diff --git a/...rg/opencadc/caom2/metasync/Harvester.java → ...ava/org/opencadc/darkspire/Harvester.java b/...rg/opencadc/caom2/metasync/Harvester.java → ...ava/org/opencadc/darkspire/Harvester.java
@@ -67,7 +67,7 @@
  ************************************************************************
  */
 
-package org.opencadc.caom2.metasync;
+package org.opencadc.darkspire;
 
 import ca.nrc.cadc.caom2.harvester.state.HarvestStateDAO;
 import ca.nrc.cadc.caom2.harvester.state.PostgresqlHarvestStateDAO;

diff --git a/...adc/caom2/metasync/HarvesterResource.java → ...opencadc/darkspire/HarvesterResource.java b/...adc/caom2/metasync/HarvesterResource.java → ...opencadc/darkspire/HarvesterResource.java
@@ -65,7 +65,7 @@
  ************************************************************************
  */
 
-package org.opencadc.caom2.metasync;
+package org.opencadc.darkspire;
 
 import java.net.URI;
 import org.apache.log4j.Logger;

diff --git a/...ava/org/opencadc/caom2/metasync/Main.java → ...ain/java/org/opencadc/darkspire/Main.java b/...ava/org/opencadc/caom2/metasync/Main.java → ...ain/java/org/opencadc/darkspire/Main.java
@@ -67,7 +67,7 @@
  ************************************************************************
  */
 
-package org.opencadc.caom2.metasync;
+package org.opencadc.darkspire;
 
 import ca.nrc.cadc.auth.RunnableAction;
 import ca.nrc.cadc.auth.SSLUtil;
@@ -92,7 +92,7 @@
 public class Main {
     private static final Logger log = Logger.getLogger(Main.class);
 
-    private static final String CONFIG_FILE_NAME = "caom2-meta-sync.properties";
+    private static final String CONFIG_FILE_NAME = "darkspire.properties";
     private static final String CERTIFICATE_FILE_LOCATION = System.getProperty("user.home") + "/.ssl/cadcproxy.pem";
     private static final String CONFIG_PREFIX = Main.class.getPackage().getName();
     private static final String LOGGING_CONFIG_KEY = CONFIG_PREFIX + ".logging";

diff --git a/...metasync/MismatchedChecksumException.java → ...arkspire/MismatchedChecksumException.java b/...metasync/MismatchedChecksumException.java → ...arkspire/MismatchedChecksumException.java
@@ -67,7 +67,7 @@
  ************************************************************************
  */
 
-package org.opencadc.caom2.metasync;
+package org.opencadc.darkspire;
 
 public class MismatchedChecksumException extends Exception {
 

diff --git a/.../caom2/metasync/ObservationHarvester.java → ...ncadc/darkspire/ObservationHarvester.java b/.../caom2/metasync/ObservationHarvester.java → ...ncadc/darkspire/ObservationHarvester.java
@@ -67,17 +67,15 @@
  ************************************************************************
  */
 
-package org.opencadc.caom2.metasync;
+package org.opencadc.darkspire;
 
 import ca.nrc.cadc.caom2.Artifact;
 import ca.nrc.cadc.caom2.Observation;
 import ca.nrc.cadc.caom2.ObservationResponse;
 import ca.nrc.cadc.caom2.ObservationState;
 import ca.nrc.cadc.caom2.ObservationURI;
 import ca.nrc.cadc.caom2.Plane;
-import ca.nrc.cadc.caom2.ac.ReadAccessGenerator;
 import ca.nrc.cadc.caom2.compute.CaomWCSValidator;
-import ca.nrc.cadc.caom2.compute.ComputeUtil;
 import ca.nrc.cadc.caom2.harvester.state.HarvestSkipURI;
 import ca.nrc.cadc.caom2.harvester.state.HarvestSkipURIDAO;
 import ca.nrc.cadc.caom2.harvester.state.HarvestState;
@@ -128,7 +126,6 @@ public class ObservationHarvester extends Harvester {
     private HarvestSkipURIDAO harvestSkipDAO;
     private boolean skipped;
     private boolean computePlaneMetadata;
-    private ReadAccessGenerator acGenerator;
     private boolean ready = false;
     private int ingested = 0;
 
@@ -145,88 +142,14 @@ public void setSkipped(boolean skipped) {
         this.skipped = skipped;
     }
 
-    public boolean getComputePlaneMetadata() {
-        return this.computePlaneMetadata;
-    }
-
-    public void setComputePlaneMetadata(boolean computePlaneMetadata) {
-        this.computePlaneMetadata = computePlaneMetadata;
-    }
-
-    public void setGenerateReadAccessTuples(File config) {
-        try {
-            Properties props = new Properties();
-            props.load(new FileReader(config));
-            String line = props.getProperty(collection);
-            if (line == null) {
-                throw new IllegalArgumentException("CONFIG: collection not found: " + collection);
-            }
-
-            Map<String,Object> groupConfig = new HashMap<>();
-            String[] parts = line.split("[ \t]+"); // one or more spaces and tabs
-            for (int i = 0; i < parts.length; i++) {
-                String option = parts[i]; // key=value pair
-                String[] kv = option.split("=");
-                if (kv.length != 2) {
-                    throw new IllegalArgumentException("invalid key=value pair: " + option);
-                }
-                if (ReadAccessGenerator.PROPOSAL_GROUP_KEY.equals(kv[0])) {
-                    boolean proposalGroup = "true".equals(kv[1]);
-                    if (proposalGroup) {
-                        groupConfig.put(ReadAccessGenerator.PROPOSAL_GROUP_KEY, proposalGroup);
-                    }
-                } else if (ReadAccessGenerator.OPERATOR_GROUP_KEY.equals(kv[0])) {
-                    String og = kv[1];
-                    if (og != null) {
-                        URI ouri = new URI(og);
-                        groupConfig.put(ReadAccessGenerator.OPERATOR_GROUP_KEY, ouri);
-                    }
-                } else if (ReadAccessGenerator.STAFF_GROUP_KEY.equals(kv[0])) {
-                    String sg = kv[1];
-                    if (sg != null) {
-                        URI suri = new URI(sg);
-                        groupConfig.put(ReadAccessGenerator.STAFF_GROUP_KEY, suri);
-                    }
-                }
-            }
-            for (Map.Entry<String,Object> me : groupConfig.entrySet()) {
-                log.debug("generate config for " + collection + ": " + me.getKey() + " = " + me.getValue());
-            }
-            this.acGenerator = new ReadAccessGenerator(collection, groupConfig);
-        } catch (IOException ex) {
-            throw new RuntimeException("failed to read config from " + config, ex);
-        } catch (Exception ex) {
-            throw new RuntimeException("CONFIG: invalid config " + config, ex);
-        } 
-    }
-
     public int getIngested() {
         return this.ingested;
     }
 
     private void init(int nthreads) {
-        if (src.getResourceType() == HarvesterResource.SOURCE_DB) {
-            Map<String, Object> srcConfig = getConfigDAO(src);
-            ConnectionConfig srcConnectionConfig = new ConnectionConfig(null, null,
-                    src.getUsername(), src.getPassword(), HarvesterResource.POSTGRESQL_DRIVER, src.getJdbcUrl());
-            final String srcDS = "jdbc/obsHarvestSrc";
-            try {
-                DBUtil.createJNDIDataSource(srcDS, srcConnectionConfig);
-            } catch (NamingException e) {
-                throw new IllegalStateException(String.format("Error creating source JNDI datasource for %s reason: %s",
-                        src, e.getMessage()));
-            }
-            srcConfig.put("jndiDataSourceName", srcDS);
-            this.srcObservationDAO = new ObservationDAO();
-            srcObservationDAO.setConfig(srcConfig);
-            ready = true;
-        } else if (src.getResourceType() == HarvesterResource.SOURCE_URI) {
-            this.srcObservationService = new RepoClient(src.getResourceID(), nthreads);
-        } else {
-            throw new IllegalStateException("BUG: unexpected HarvestResource resource type: " + src);
-        }
+        this.srcObservationService = new RepoClient(src.getResourceID(), nthreads);
 
-        // for now, dest is always a database
+        // dest is always a database
         Map<String, Object> destConfig = getConfigDAO(dest);
         ConnectionConfig destConnectionConfig = new ConnectionConfig(null, null,
                 dest.getUsername(), dest.getPassword(), HarvesterResource.POSTGRESQL_DRIVER, dest.getJdbcUrl());
@@ -249,12 +172,10 @@ private void init(int nthreads) {
         }
         initHarvestState(destObservationDAO.getDataSource(), Observation.class);
 
-        if (srcObservationService != null) {
-            if (srcObservationService.isObsAvailable()) {
-                ready = true;
-            } else {
-                log.error("Not available obs endpoint in " + srcObservationService.toString());
-            }
+        if (srcObservationService.isObsAvailable()) {
+            ready = true;
+        } else {
+            log.error("Not available: obs endpoint in " + srcObservationService.toString());
         }
     }
 

diff --git a/...adc/caom2/metasync/SkippedWrapperURI.java → ...opencadc/darkspire/SkippedWrapperURI.java b/...adc/caom2/metasync/SkippedWrapperURI.java → ...opencadc/darkspire/SkippedWrapperURI.java
@@ -67,7 +67,7 @@
  ************************************************************************
  */
 
-package org.opencadc.caom2.metasync;
+package org.opencadc.darkspire;
 
 import ca.nrc.cadc.caom2.harvester.state.HarvestSkipURI;
 import org.apache.log4j.Logger;