Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
* first version * Dspace harvester first version * harvest other entities + dspace docker compose * Fix issues in the harvest alg * Add logging, fix OAI issue en Col/Com/Rep , port RESTv7 harvester Co-authored-by: Jose Ortiz <jose.ortizv@ucuenca.edu.ec>
- Loading branch information
Showing
32 changed files
with
1,977 additions
and
83 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,77 +1,112 @@ | ||
| <project xmlns="http://maven.apache.org/POM/4.0.0" | ||
| xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" | ||
| xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> | ||
| <modelVersion>4.0.0</modelVersion> | ||
| <artifactId>org.vivoweb.dspacevivo.transformation</artifactId> | ||
| <licenses> | ||
| <license> | ||
| <name>BSD 3-Clause License</name> | ||
| <url>https://raw.githubusercontent.com/vivo-project/VIVO/develop/LICENSE</url> | ||
| <distribution>repo</distribution> | ||
| </license> | ||
| </licenses> | ||
| <developers> | ||
| <developer> | ||
| <name>Michel Héon</name> | ||
| <organization>Université du Québec à Montréal</organization> | ||
| <organizationUrl>http://uqam.ca/</organizationUrl> | ||
| <url>https://ca.linkedin.com/in/michelheon</url> | ||
| </developer> | ||
| </developers> | ||
| <properties> | ||
| </properties> | ||
| <build> | ||
| </build> | ||
| <parent> | ||
| <groupId>org.vivoweb.dspacevivo</groupId> | ||
| <artifactId>org.vivoweb.dspacevivo.installer</artifactId> | ||
| <version>0.0.1-SNAPSHOT</version> | ||
| <relativePath>../../releng/org.vivoweb.dspacevivo.installer</relativePath> | ||
| </parent> | ||
| <dependencies> | ||
| <dependency> | ||
| <groupId>org.apache.jena</groupId> | ||
| <artifactId>jena-core</artifactId> | ||
| </dependency> | ||
| <dependency> | ||
| <groupId>org.apache.jena</groupId> | ||
| <artifactId>apache-jena-libs</artifactId> | ||
| <type>pom</type> | ||
| </dependency> | ||
| <dependency> | ||
| <groupId>org.apache.jena</groupId> | ||
| <artifactId>jena-arq</artifactId> | ||
| </dependency> | ||
| <dependency> | ||
| <groupId>org.apache.jena</groupId> | ||
| <artifactId>jena-cmds</artifactId> | ||
| </dependency> | ||
| <dependency> | ||
| <groupId>org.apache.jena</groupId> | ||
| <artifactId>apache-jena</artifactId> | ||
| <type>pom</type> | ||
| </dependency> | ||
| <dependency> | ||
| <groupId>com.fasterxml.jackson.core</groupId> | ||
| <artifactId>jackson-core</artifactId> | ||
| </dependency> | ||
| <dependency> | ||
| <groupId>com.fasterxml.jackson.core</groupId> | ||
| <artifactId>jackson-databind</artifactId> | ||
| </dependency> | ||
| <dependency> | ||
| <groupId>com.fasterxml.jackson.core</groupId> | ||
| <artifactId>jackson-annotations</artifactId> | ||
| </dependency> | ||
| <dependency> | ||
| <groupId>org.vivoweb.dspacevivo</groupId> | ||
| <artifactId>org.vivoweb.dspacevivo.model.ontologie</artifactId> | ||
| <version>0.0.1-SNAPSHOT</version> | ||
| </dependency> | ||
| <dependency> | ||
| <groupId>org.vivoweb.dspacevivo</groupId> | ||
| <artifactId>org.vivoweb.dspacevivo.model.openapi</artifactId> | ||
| <version>0.0.1-SNAPSHOT</version> | ||
| </dependency> | ||
| </dependencies> | ||
| xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" | ||
| xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> | ||
| <modelVersion>4.0.0</modelVersion> | ||
| <artifactId>org.vivoweb.dspacevivo.transformation</artifactId> | ||
| <licenses> | ||
| <license> | ||
| <name>BSD 3-Clause License</name> | ||
| <url>https://raw.githubusercontent.com/vivo-project/VIVO/develop/LICENSE</url> | ||
| <distribution>repo</distribution> | ||
| </license> | ||
| </licenses> | ||
| <developers> | ||
| <developer> | ||
| <name>Michel Héon</name> | ||
| <organization>Université du Québec à Montréal</organization> | ||
| <organizationUrl>http://uqam.ca/</organizationUrl> | ||
| <url>https://ca.linkedin.com/in/michelheon</url> | ||
| </developer> | ||
| </developers> | ||
| <properties> | ||
| </properties> | ||
| <build> | ||
| </build> | ||
| <parent> | ||
| <groupId>org.vivoweb.dspacevivo</groupId> | ||
| <artifactId>org.vivoweb.dspacevivo.installer</artifactId> | ||
| <version>0.0.1-SNAPSHOT</version> | ||
| <relativePath>../../releng/org.vivoweb.dspacevivo.installer</relativePath> | ||
| </parent> | ||
| <dependencies> | ||
| <dependency> | ||
| <groupId>com.konghq</groupId> | ||
| <artifactId>unirest-java</artifactId> | ||
| <version>3.13.6</version> | ||
| </dependency> | ||
| <dependency> | ||
| <groupId>org.apache.logging.log4j</groupId> | ||
| <artifactId>log4j-api</artifactId> | ||
| <version>2.17.0</version> | ||
| </dependency> | ||
| <dependency> | ||
| <groupId>org.apache.logging.log4j</groupId> | ||
| <artifactId>log4j-core</artifactId> | ||
| <version>2.17.0</version> | ||
| </dependency> | ||
| <dependency> | ||
| <groupId>org.apache.logging.log4j</groupId> | ||
| <artifactId>log4j-slf4j-impl</artifactId> | ||
| <version>2.17.0</version> | ||
| </dependency> | ||
| <dependency> | ||
| <groupId>org.jsoup</groupId> | ||
| <artifactId>jsoup</artifactId> | ||
| <version>1.13.1</version> | ||
| </dependency> | ||
| <dependency> | ||
| <groupId>commons-io</groupId> | ||
| <artifactId>commons-io</artifactId> | ||
| <version>2.11.0</version> | ||
| </dependency> | ||
| <dependency> | ||
| <groupId>org.glassfish</groupId> | ||
| <artifactId>javax.el</artifactId> | ||
| <version>3.0.0</version> | ||
| </dependency> | ||
| <dependency> | ||
| <groupId>org.apache.jena</groupId> | ||
| <artifactId>jena-core</artifactId> | ||
| </dependency> | ||
| <dependency> | ||
| <groupId>org.apache.jena</groupId> | ||
| <artifactId>apache-jena-libs</artifactId> | ||
| <type>pom</type> | ||
| </dependency> | ||
| <dependency> | ||
| <groupId>org.apache.jena</groupId> | ||
| <artifactId>jena-arq</artifactId> | ||
| </dependency> | ||
| <dependency> | ||
| <groupId>org.apache.jena</groupId> | ||
| <artifactId>jena-cmds</artifactId> | ||
| </dependency> | ||
| <dependency> | ||
| <groupId>org.apache.jena</groupId> | ||
| <artifactId>apache-jena</artifactId> | ||
| <type>pom</type> | ||
| </dependency> | ||
| <dependency> | ||
| <groupId>com.fasterxml.jackson.core</groupId> | ||
| <artifactId>jackson-core</artifactId> | ||
| </dependency> | ||
| <dependency> | ||
| <groupId>com.fasterxml.jackson.core</groupId> | ||
| <artifactId>jackson-databind</artifactId> | ||
| </dependency> | ||
| <dependency> | ||
| <groupId>com.fasterxml.jackson.core</groupId> | ||
| <artifactId>jackson-annotations</artifactId> | ||
| </dependency> | ||
| <dependency> | ||
| <groupId>org.vivoweb.dspacevivo</groupId> | ||
| <artifactId>org.vivoweb.dspacevivo.model.ontologie</artifactId> | ||
| <version>0.0.1-SNAPSHOT</version> | ||
| </dependency> | ||
| <dependency> | ||
| <groupId>org.vivoweb.dspacevivo</groupId> | ||
| <artifactId>org.vivoweb.dspacevivo.model.openapi</artifactId> | ||
| <version>0.0.1-SNAPSHOT</version> | ||
| </dependency> | ||
| </dependencies> | ||
| </project> |
28 changes: 28 additions & 0 deletions
28
...ransformation/src/main/java/org/vivoweb/dspacevivo/transformation/ConsoleApplication.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,28 @@ | ||
| package org.vivoweb.dspacevivo.transformation; | ||
|
|
||
| import java.io.IOException; | ||
| import org.slf4j.Logger; | ||
| import org.slf4j.LoggerFactory; | ||
| import org.vivoweb.dspacevivo.transformation.harvester.HarvesterRunner; | ||
|
|
||
| public class ConsoleApplication { | ||
|
|
||
| private static Logger logger = LoggerFactory.getLogger(ConsoleApplication.class); | ||
|
|
||
| public static void main(String[] args) throws IOException { | ||
|
|
||
| logger.info("Testing Dspace Harvester..."); | ||
| HarvesterRunner runner = new HarvesterRunner(); | ||
| logger.info("Init Dspace Harvester..."); | ||
| runner.init(); | ||
| logger.info("Harvesting Items..."); | ||
| runner.harvestItems(); | ||
| logger.info("Harvesting Communities..."); | ||
| //runner.harvestCommunities(); | ||
| logger.info("Harvesting Collections..."); | ||
| //runner.harvestCollections(); | ||
| logger.info("Harvesting Repositories..."); | ||
| //runner.harvestRepositories(); | ||
| } | ||
|
|
||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
31 changes: 31 additions & 0 deletions
31
...mation/src/main/java/org/vivoweb/dspacevivo/transformation/harvester/DspaceHarvester.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,31 @@ | ||
| package org.vivoweb.dspacevivo.transformation.harvester; | ||
|
|
||
| import java.util.Iterator; | ||
| import java.util.Properties; | ||
| import org.vivoweb.dspacevivo.model.Collection; | ||
| import org.vivoweb.dspacevivo.model.Community; | ||
| import org.vivoweb.dspacevivo.model.Item; | ||
| import org.vivoweb.dspacevivo.model.Repository; | ||
|
|
||
| public abstract class DspaceHarvester { | ||
|
|
||
| protected Properties conf = null; | ||
|
|
||
| public DspaceHarvester(Properties conf) { | ||
| this.conf = conf; | ||
| } | ||
|
|
||
| public Properties getConf() { | ||
| return conf; | ||
| } | ||
|
|
||
| public abstract void connect(); | ||
|
|
||
| public abstract Iterator<Item> harvestItems(); | ||
|
|
||
| public abstract Iterator<Community> harvestCommunity(); | ||
|
|
||
| public abstract Iterator<Collection> harvestCollection(); | ||
|
|
||
| public abstract Iterator<Repository> harvestRepository(); | ||
| } |
Oops, something went wrong.