NAS-2468: Database upgrade sql added for postgres and mysql.

netarchivesuite · Dec 21, 2015 · 9937483 · 9937483
1 parent 83cf7b9
commit 9937483
Show file tree

Hide file tree

Showing 13 changed files with 275 additions and 16 deletions.
diff --git a/deploy/deploy-core/scripts/derby/migration/eav_attribute.1.sql b/deploy/deploy-core/scripts/derby/migration/eav_attribute.1.sql
@@ -0,0 +1,13 @@
+CREATE TABLE eav_attribute (
+	tree_id INTEGER NOT NULL,
+	id INTEGER NOT NULL generated always as identity,
+	entity_id INTEGER NOT NULL,
+	type_id INTEGER NOT NULL,
+	val_int INTEGER,
+	val_datetime TIMESTAMP,
+	val_varchar VARCHAR(8000),
+	val_text CLOB,
+	PRIMARY KEY (tree_id, id)
+);
+
+CREATE UNIQUE INDEX eav_attribute_idx on eav_attribute(tree_id, entity_id);
diff --git a/deploy/deploy-core/scripts/derby/migration/eav_type_attribute.1.sql b/deploy/deploy-core/scripts/derby/migration/eav_type_attribute.1.sql
@@ -0,0 +1,16 @@
+CREATE TABLE eav_type_attribute (
+	tree_id INTEGER NOT NULL,
+	id INTEGER NOT NULL,
+	name VARCHAR(96) NOT NULL,
+	class_namespace VARCHAR(96) NOT NULL,
+	class_name VARCHAR(96) NOT NULL,
+	datatype INTEGER NOT NULL,
+	viewtype INTEGER NOT NULL,
+	def_int INTEGER,
+	def_datetime TIMESTAMP,
+	def_varchar VARCHAR(8000),
+	def_text CLOB,
+	PRIMARY KEY (tree_id, id)
+);
+
+CREATE UNIQUE INDEX eav_type_attribute_idx on eav_type_attribute(tree_id, id);
diff --git a/deploy/deploy-core/scripts/postgresql/migration/eav_attribute.1.sql b/deploy/deploy-core/scripts/postgresql/migration/eav_attribute.1.sql
@@ -0,0 +1,16 @@
+CREATE SEQUENCE eav_attribute_seq;
+
+CREATE TABLE eav_attribute (
+	tree_id INTEGER NOT NULL,
+	id INTEGER NOT NULL DEFAULT NEXTVAL('eav_attribute_seq'),
+	entity_id INTEGER NOT NULL,
+	type_id INTEGER NOT NULL,
+	val_int INTEGER NULL,
+	val_datetime TIMESTAMP NULL,
+	val_varchar VARCHAR(8000) NULL,
+	val_text TEXT NULL
+);
+
+ALTER TABLE eav_attribute ADD CONSTRAINT eav_attribute_pkey PRIMARY KEY (tree_id, id);
+
+CREATE UNIQUE INDEX eav_attribute_idx on eav_attribute(tree_id, entity_id) TABLESPACE tsindex;
diff --git a/deploy/deploy-core/scripts/postgresql/migration/eav_type_attribute.1.sql b/deploy/deploy-core/scripts/postgresql/migration/eav_type_attribute.1.sql
@@ -0,0 +1,17 @@
+CREATE TABLE eav_type_attribute (
+	tree_id INTEGER NOT NULL,
+	id INTEGER NOT NULL,
+	name VARCHAR(96) NOT NULL,
+	class_namespace VARCHAR(96) NOT NULL,
+	class_name VARCHAR(96) NOT NULL,
+	datatype INTEGER NOT NULL,
+	viewtype INTEGER NOT NULL,
+	def_int INTEGER NULL,
+	def_datetime TIMESTAMP NULL,
+	def_varchar VARCHAR(8000) NULL,
+	def_text TEXT NULL
+);
+
+ALTER TABLE eav_type_attribute ADD CONSTRAINT eav_type_attribute_pkey PRIMARY KEY (tree_id, id);
+
+CREATE UNIQUE INDEX eav_type_attribute_idx on eav_type_attribute(tree_id, id) TABLESPACE tsindex;
diff --git a/harvester/harvester-core/pom.xml b/harvester/harvester-core/pom.xml
@@ -133,13 +133,6 @@
     <dependency>
       <groupId>com.antiaction</groupId>
       <artifactId>raptor-dbl</artifactId>
-      <version>0.1.0-RC1</version>
-      <exclusions>
-        <exclusion>
-          <groupId>*</groupId>
-          <artifactId>*</artifactId>
-        </exclusion>
-      </exclusions>
     </dependency>
 
     <dependency>
@@ -150,6 +143,24 @@
   </dependencies>
 
   <build>
+    <resources>
+      <resource>
+        <directory>src/main/resources</directory>
+      </resource>
+      <resource>
+        <directory>${project.basedir}/../../deploy/deploy-core/scripts/postgresql/migration/</directory>
+        <targetPath>sql-migration/postgresql/</targetPath>
+      </resource>
+      <resource>
+        <directory>${project.basedir}/../../deploy/deploy-core/scripts/mysql/migration/</directory>
+        <targetPath>sql-migration/mysql/</targetPath>
+      </resource>
+      <resource>
+        <directory>${project.basedir}/../../deploy/deploy-core/scripts/derby/migration/</directory>
+        <targetPath>sql-migration/derby/</targetPath>
+      </resource>
+    </resources>
+
     <plugins>
       <plugin>
         <groupId>org.apache.maven.plugins</groupId>

diff --git a/harvester/harvester-core/src/main/java/dk/netarkivet/harvester/datamodel/DBSpecifics.java b/harvester/harvester-core/src/main/java/dk/netarkivet/harvester/datamodel/DBSpecifics.java
@@ -155,8 +155,12 @@ public synchronized void updateTable(String tableName, int toVersion) {
             upgradeOrderTemplatesTable(currentVersion, toVersion);
         } else if (tableName.equals(HarvesterDatabaseTables.HARVESTCHANNELS.getTablename())) {
             upgradeHarvestchannelTable(currentVersion, toVersion);
-            // Add new if else when other tables need to be upgraded
+        } else if (tableName.equals(HarvesterDatabaseTables.EAVTYPEATTRIBUTE.getTablename())) {
+            upgradeEavTypeAttributeTable(currentVersion, toVersion);
+        } else if (tableName.equals(HarvesterDatabaseTables.EAVATTRIBUTE.getTablename())) {
+            upgradeEavAttributeTable(currentVersion, toVersion);
         } else {
+            // Add new if else when other tables need to be upgraded
             throw new NotImplementedException("No method exists for migrating table '" + tableName + "' to version "
                     + toVersion);
         }
@@ -691,4 +695,48 @@ public void updateTables() {
         }
     }
 
+    /**
+     * Migrate the eavtypeattribute table.
+     * @param currentVersion the current version of the eavtypeattribute table
+     * @param toVersion the required version of the eavtypeattribute table
+     */
+    public void upgradeEavTypeAttributeTable(int currentVersion, int toVersion) {
+        if (currentVersion == 0 && toVersion >= 1) {
+        	createEavTypeAttributeTable(1);
+            currentVersion = 1;
+        }
+        if (currentVersion > HarvesterDatabaseTables.EAVTYPEATTRIBUTE.getRequiredVersion()) {
+            throw new NotImplementedException("No method exists for migrating table '"
+                    + HarvesterDatabaseTables.EAVTYPEATTRIBUTE.getTablename() + "' from version " + currentVersion
+                    + " to version " + toVersion);
+        }
+    }
+
+    /**
+     * Create the EavTypeAttribute table in the database.
+     */
+    public abstract void createEavTypeAttributeTable(int toVersion);
+
+    /**
+     * Migrate the eavattribute table.
+     * @param currentVersion the current version of the eavattribute table
+     * @param toVersion the required version of the eavattribute table
+     */
+    public void upgradeEavAttributeTable(int currentVersion, int toVersion) {
+        if (currentVersion == 0 && toVersion >= 1) {
+        	createEavAttributeTable(1);
+            currentVersion = 1;
+        }
+        if (currentVersion > HarvesterDatabaseTables.EAVATTRIBUTE.getRequiredVersion()) {
+            throw new NotImplementedException("No method exists for migrating table '"
+                    + HarvesterDatabaseTables.EAVATTRIBUTE.getTablename() + "' from version " + currentVersion
+                    + " to version " + toVersion);
+        }
+    }
+
+    /**
+     * Create the EavAttributeTable table in the database.
+     */
+    public abstract void createEavAttributeTable(int toVersion);
+
 }
diff --git a/harvester/harvester-core/src/main/java/dk/netarkivet/harvester/datamodel/DerbySpecifics.java b/harvester/harvester-core/src/main/java/dk/netarkivet/harvester/datamodel/DerbySpecifics.java
@@ -390,10 +390,23 @@ protected void migrateExtendedFieldTableValueV1toV2() {
         HarvestDBConnection.updateTable("extendedfieldvalue", 2, sqlStatements);
     }
 
-    @Override protected void migrateOrderTemplatesTablev1tov2() {
+    @Override
+    protected void migrateOrderTemplatesTablev1tov2() {
         String tableName = HarvesterDatabaseTables.ORDERTEMPLATES.getTablename();
         String[] sqlStatements = {"ALTER TABLE " + tableName + " ADD COLUMN isActive BOOLEAN NOT NULL DEFAULT TRUE"};
         HarvestDBConnection.updateTable(tableName, 2, sqlStatements);
     }
 
+    @Override
+    public void createEavTypeAttributeTable(int toVersion) {
+        String tableName = HarvesterDatabaseTables.EAVTYPEATTRIBUTE.getTablename();
+        HarvestDBConnection.executeSql("derby", tableName, 1 );
+    }
+
+    @Override
+    public void createEavAttributeTable(int toVersion) {
+        String tableName = HarvesterDatabaseTables.EAVATTRIBUTE.getTablename();
+        HarvestDBConnection.executeSql("derby", tableName, 1 );
+    }
+
 }
diff --git a/...r/harvester-core/src/main/java/dk/netarkivet/harvester/datamodel/HarvestDBConnection.java b/...r/harvester-core/src/main/java/dk/netarkivet/harvester/datamodel/HarvestDBConnection.java
@@ -24,12 +24,17 @@
 package dk.netarkivet.harvester.datamodel;
 
 import java.beans.PropertyVetoException;
+import java.io.IOException;
+import java.io.InputStream;
 import java.sql.Connection;
 import java.sql.SQLException;
+import java.util.List;
+import java.util.Map;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import com.antiaction.raptor.sql.ExecuteSqlFile;
 import com.mchange.v2.c3p0.ComboPooledDataSource;
 
 import dk.netarkivet.common.CommonSettings;
@@ -117,7 +122,6 @@ public static synchronized Connection get() {
      * @throws IOFailure in case of problems in interacting with the database
      */
     protected static void updateTable(final String table, final int newVersion, final String... updates) {
-
         Connection c = get();
         updateTable(c, table, newVersion, updates);
     }
@@ -143,6 +147,27 @@ public static void updateTable(Connection c, final String table, final int newVe
         }
     }
 
+    protected static void updateTableVersion(final String table, final int newVersion, final String... updates) {
+        Connection c = get();
+        updateTableVersion(c, table, newVersion);
+    }
+
+    public static void updateTableVersion(Connection c, final String table, final int newVersion) {
+        log.info("Updating table '{}' to version {}", table, newVersion);
+        String updateSchemaversionSql = null;
+        if (newVersion == 1) {
+            updateSchemaversionSql = "INSERT INTO schemaversions(tablename, version) VALUES ('" + table + "', 1)";
+        } else {
+            updateSchemaversionSql = "UPDATE schemaversions SET version = " + newVersion + " WHERE tablename = '"
+                    + table + "'";
+        }
+        try {
+            DBUtils.executeSQL(c, updateSchemaversionSql);
+        } finally {
+            release(c);
+        }
+    }
+
     /**
      * Method for retrieving the url for the harvest definition database. This url will be constructed from the
      * base-url, the machine, the port and the directory. If the database is internal, then only the base-url should
@@ -279,4 +304,35 @@ private static void initDataSource(DBSpecifics dbSpec, String jdbcUrl) throws SQ
                     dataSource.getPreferredTestQuery(), dataSource.isTestConnectionOnCheckin());
         }
     }
+
+    public static void executeSql(String dbm, String tableName, int version) {
+    	Connection conn = HarvestDBConnection.get();
+        executeSql(conn, dbm, tableName, version);
+        HarvestDBConnection.release(conn);
+        conn = null;
+    }
+
+    public static void executeSql(Connection conn, String dbm, String tableName, int version) {
+    	InputStream in = DerbySpecifics.class.getClassLoader().getResourceAsStream("sql-migration/" + dbm +"/" + tableName + "." + version + ".sql");
+    	try {
+        	List<Map.Entry<String, String>> statements = ExecuteSqlFile.splitSql(in, "UTF-8", 8192);
+        	in.close();
+        	in = null;
+            ExecuteSqlFile.executeStatements(conn, statements);
+            HarvestDBConnection.updateTable(conn, tableName, version);
+    	} catch (IOException e) {
+    		e.printStackTrace();
+    	} catch (SQLException e) {
+    		e.printStackTrace();
+    	} finally {
+    		if (in != null) {
+    			try {
+        			in.close();
+    			} catch (IOException e) {
+    			}
+    			in = null;
+    		}
+    	}
+    }
+
 }
diff --git a/...rvester-core/src/main/java/dk/netarkivet/harvester/datamodel/HarvesterDatabaseTables.java b/...rvester-core/src/main/java/dk/netarkivet/harvester/datamodel/HarvesterDatabaseTables.java
@@ -409,6 +409,36 @@ public int getRequiredVersion() {
             return REQUIRED_VERSION;
         }
 
+        @Override
+        public String getTablename() {
+            return NAME;
+        }
+    },
+    /** Harvest channels. */
+    EAVTYPEATTRIBUTE {
+        static final String NAME = "eav_type_attribute";
+        static final int REQUIRED_VERSION = 1;
+
+        @Override
+        public int getRequiredVersion() {
+            return REQUIRED_VERSION;
+        }
+
+        @Override
+        public String getTablename() {
+            return NAME;
+        }
+    },
+    /** Harvest channels. */
+    EAVATTRIBUTE {
+        static final String NAME = "eav_attribute";
+        static final int REQUIRED_VERSION = 1;
+
+        @Override
+        public int getRequiredVersion() {
+            return REQUIRED_VERSION;
+        }
+
         @Override
         public String getTablename() {
             return NAME;

diff --git a/harvester/harvester-core/src/main/java/dk/netarkivet/harvester/datamodel/MySQLSpecifics.java b/harvester/harvester-core/src/main/java/dk/netarkivet/harvester/datamodel/MySQLSpecifics.java
@@ -410,4 +410,16 @@ protected void migrateExtendedFieldTableValueV1toV2() {
         HarvestDBConnection.updateTable(tableName, 2, sqlStatements);
     }
 
+    @Override
+    public void createEavTypeAttributeTable(int toVersion) {
+        String tableName = HarvesterDatabaseTables.EAVTYPEATTRIBUTE.getTablename();
+        HarvestDBConnection.executeSql("mysql", tableName, 1 );
+    }
+
+    @Override
+    public void createEavAttributeTable(int toVersion) {
+        String tableName = HarvesterDatabaseTables.EAVATTRIBUTE.getTablename();
+        HarvestDBConnection.executeSql("mysql", tableName, 1 );
+    }
+
 }
diff --git a/...r/harvester-core/src/main/java/dk/netarkivet/harvester/datamodel/PostgreSQLSpecifics.java b/...r/harvester-core/src/main/java/dk/netarkivet/harvester/datamodel/PostgreSQLSpecifics.java
@@ -345,11 +345,23 @@ protected void migrateExtendedFieldTableValueV1toV2() {
         HarvestDBConnection.updateTable("extendedfieldvalue", 2, sqlStatements);
     }
 
-
-    @Override protected void migrateOrderTemplatesTablev1tov2() {
+    @Override
+    protected void migrateOrderTemplatesTablev1tov2() {
         final String tablename = HarvesterDatabaseTables.ORDERTEMPLATES.getTablename();
         String[] sqlStatements = {"ALTER TABLE " + tablename + " ADD COLUMN isActive BOOL NOT NULL DEFAULT TRUE"};
         HarvestDBConnection.updateTable(tablename, 2, sqlStatements);
     }
 
+    @Override
+    public void createEavTypeAttributeTable(int toVersion) {
+        String tableName = HarvesterDatabaseTables.EAVTYPEATTRIBUTE.getTablename();
+        HarvestDBConnection.executeSql("postgresql", tableName, 1 );
+    }
+
+    @Override
+    public void createEavAttributeTable(int toVersion) {
+        String tableName = HarvesterDatabaseTables.EAVATTRIBUTE.getTablename();
+        HarvestDBConnection.executeSql("postgresql", tableName, 1 );
+    }
+
 }
diff --git a/...er-core/src/main/java/dk/netarkivet/harvester/webinterface/SnapshotHarvestDefinition.java b/...er-core/src/main/java/dk/netarkivet/harvester/webinterface/SnapshotHarvestDefinition.java
@@ -200,9 +200,12 @@ public void processRequest(PageContext context, I18n i18n) {
 
         // EAV
         try {
-        	long entity_id = hd.getOid();
-            EAV eav = eavDAOProvider.get();
-            List<AttributeAndType> attributesAndTypes = eav.getAttributesAndTypes(EAV.SNAPSHOT_TREE_ID, (int)entity_id);
+        	Long entity_id = hd.getOid();
+        	if (entity_id == null) {
+        		entity_id = 0L;
+        	}
+        	EAV eav = eavDAOProvider.get();
+            List<AttributeAndType> attributesAndTypes = eav.getAttributesAndTypes(EAV.SNAPSHOT_TREE_ID, (int)((long)entity_id));
             AttributeAndType attributeAndType;
             AttributeTypeBase attributeType;
             AttributeBase attribute;
@@ -212,7 +215,7 @@ public void processRequest(PageContext context, I18n i18n) {
             	attribute = attributeAndType.attribute;
             	if (attribute == null) {
                 	attribute = attributeType.instanceOf();
-                	attribute.entity_id = (int)entity_id;
+                	attribute.entity_id = (int)((long)entity_id);
             	}
             	switch (attributeType.viewtype) {
             	case 1: