diff --git a/deploy/deploy-core/scripts/derby/migration/eav_attribute.1.sql b/deploy/deploy-core/scripts/derby/migration/eav_attribute.1.sql new file mode 100644 index 0000000000..6936ebf830 --- /dev/null +++ b/deploy/deploy-core/scripts/derby/migration/eav_attribute.1.sql @@ -0,0 +1,13 @@ +CREATE TABLE eav_attribute ( + tree_id INTEGER NOT NULL, + id INTEGER NOT NULL generated always as identity, + entity_id INTEGER NOT NULL, + type_id INTEGER NOT NULL, + val_int INTEGER, + val_datetime TIMESTAMP, + val_varchar VARCHAR(8000), + val_text CLOB, + PRIMARY KEY (tree_id, id) +); + +CREATE UNIQUE INDEX eav_attribute_idx on eav_attribute(tree_id, entity_id); diff --git a/deploy/deploy-core/scripts/derby/migration/eav_type_attribute.1.sql b/deploy/deploy-core/scripts/derby/migration/eav_type_attribute.1.sql new file mode 100644 index 0000000000..4fe51f6532 --- /dev/null +++ b/deploy/deploy-core/scripts/derby/migration/eav_type_attribute.1.sql @@ -0,0 +1,16 @@ +CREATE TABLE eav_type_attribute ( + tree_id INTEGER NOT NULL, + id INTEGER NOT NULL, + name VARCHAR(96) NOT NULL, + class_namespace VARCHAR(96) NOT NULL, + class_name VARCHAR(96) NOT NULL, + datatype INTEGER NOT NULL, + viewtype INTEGER NOT NULL, + def_int INTEGER, + def_datetime TIMESTAMP, + def_varchar VARCHAR(8000), + def_text CLOB, + PRIMARY KEY (tree_id, id) +); + +CREATE UNIQUE INDEX eav_type_attribute_idx on eav_type_attribute(tree_id, id); diff --git a/deploy/deploy-core/scripts/postgresql/migration/eav_attribute.1.sql b/deploy/deploy-core/scripts/postgresql/migration/eav_attribute.1.sql new file mode 100644 index 0000000000..35eeb61f58 --- /dev/null +++ b/deploy/deploy-core/scripts/postgresql/migration/eav_attribute.1.sql @@ -0,0 +1,16 @@ +CREATE SEQUENCE eav_attribute_seq; + +CREATE TABLE eav_attribute ( + tree_id INTEGER NOT NULL, + id INTEGER NOT NULL DEFAULT NEXTVAL('eav_attribute_seq'), + entity_id INTEGER NOT NULL, + type_id INTEGER NOT NULL, + val_int INTEGER NULL, + val_datetime TIMESTAMP NULL, + val_varchar VARCHAR(8000) NULL, + val_text TEXT NULL +); + +ALTER TABLE eav_attribute ADD CONSTRAINT eav_attribute_pkey PRIMARY KEY (tree_id, id); + +CREATE UNIQUE INDEX eav_attribute_idx on eav_attribute(tree_id, entity_id) TABLESPACE tsindex; diff --git a/deploy/deploy-core/scripts/postgresql/migration/eav_type_attribute.1.sql b/deploy/deploy-core/scripts/postgresql/migration/eav_type_attribute.1.sql new file mode 100644 index 0000000000..27367bf54d --- /dev/null +++ b/deploy/deploy-core/scripts/postgresql/migration/eav_type_attribute.1.sql @@ -0,0 +1,17 @@ +CREATE TABLE eav_type_attribute ( + tree_id INTEGER NOT NULL, + id INTEGER NOT NULL, + name VARCHAR(96) NOT NULL, + class_namespace VARCHAR(96) NOT NULL, + class_name VARCHAR(96) NOT NULL, + datatype INTEGER NOT NULL, + viewtype INTEGER NOT NULL, + def_int INTEGER NULL, + def_datetime TIMESTAMP NULL, + def_varchar VARCHAR(8000) NULL, + def_text TEXT NULL +); + +ALTER TABLE eav_type_attribute ADD CONSTRAINT eav_type_attribute_pkey PRIMARY KEY (tree_id, id); + +CREATE UNIQUE INDEX eav_type_attribute_idx on eav_type_attribute(tree_id, id) TABLESPACE tsindex; diff --git a/harvester/harvester-core/pom.xml b/harvester/harvester-core/pom.xml index 198607dd31..e2bd8c8ff5 100644 --- a/harvester/harvester-core/pom.xml +++ b/harvester/harvester-core/pom.xml @@ -133,13 +133,6 @@ com.antiaction raptor-dbl - 0.1.0-RC1 - - - * - * - - @@ -150,6 +143,24 @@ + + + src/main/resources + + + ${project.basedir}/../../deploy/deploy-core/scripts/postgresql/migration/ + sql-migration/postgresql/ + + + ${project.basedir}/../../deploy/deploy-core/scripts/mysql/migration/ + sql-migration/mysql/ + + + ${project.basedir}/../../deploy/deploy-core/scripts/derby/migration/ + sql-migration/derby/ + + + org.apache.maven.plugins diff --git a/harvester/harvester-core/src/main/java/dk/netarkivet/harvester/datamodel/DBSpecifics.java b/harvester/harvester-core/src/main/java/dk/netarkivet/harvester/datamodel/DBSpecifics.java index 18dc2adada..e9f27f45a9 100644 --- a/harvester/harvester-core/src/main/java/dk/netarkivet/harvester/datamodel/DBSpecifics.java +++ b/harvester/harvester-core/src/main/java/dk/netarkivet/harvester/datamodel/DBSpecifics.java @@ -155,8 +155,12 @@ public synchronized void updateTable(String tableName, int toVersion) { upgradeOrderTemplatesTable(currentVersion, toVersion); } else if (tableName.equals(HarvesterDatabaseTables.HARVESTCHANNELS.getTablename())) { upgradeHarvestchannelTable(currentVersion, toVersion); - // Add new if else when other tables need to be upgraded + } else if (tableName.equals(HarvesterDatabaseTables.EAVTYPEATTRIBUTE.getTablename())) { + upgradeEavTypeAttributeTable(currentVersion, toVersion); + } else if (tableName.equals(HarvesterDatabaseTables.EAVATTRIBUTE.getTablename())) { + upgradeEavAttributeTable(currentVersion, toVersion); } else { + // Add new if else when other tables need to be upgraded throw new NotImplementedException("No method exists for migrating table '" + tableName + "' to version " + toVersion); } @@ -691,4 +695,48 @@ public void updateTables() { } } + /** + * Migrate the eavtypeattribute table. + * @param currentVersion the current version of the eavtypeattribute table + * @param toVersion the required version of the eavtypeattribute table + */ + public void upgradeEavTypeAttributeTable(int currentVersion, int toVersion) { + if (currentVersion == 0 && toVersion >= 1) { + createEavTypeAttributeTable(1); + currentVersion = 1; + } + if (currentVersion > HarvesterDatabaseTables.EAVTYPEATTRIBUTE.getRequiredVersion()) { + throw new NotImplementedException("No method exists for migrating table '" + + HarvesterDatabaseTables.EAVTYPEATTRIBUTE.getTablename() + "' from version " + currentVersion + + " to version " + toVersion); + } + } + + /** + * Create the EavTypeAttribute table in the database. + */ + public abstract void createEavTypeAttributeTable(int toVersion); + + /** + * Migrate the eavattribute table. + * @param currentVersion the current version of the eavattribute table + * @param toVersion the required version of the eavattribute table + */ + public void upgradeEavAttributeTable(int currentVersion, int toVersion) { + if (currentVersion == 0 && toVersion >= 1) { + createEavAttributeTable(1); + currentVersion = 1; + } + if (currentVersion > HarvesterDatabaseTables.EAVATTRIBUTE.getRequiredVersion()) { + throw new NotImplementedException("No method exists for migrating table '" + + HarvesterDatabaseTables.EAVATTRIBUTE.getTablename() + "' from version " + currentVersion + + " to version " + toVersion); + } + } + + /** + * Create the EavAttributeTable table in the database. + */ + public abstract void createEavAttributeTable(int toVersion); + } diff --git a/harvester/harvester-core/src/main/java/dk/netarkivet/harvester/datamodel/DerbySpecifics.java b/harvester/harvester-core/src/main/java/dk/netarkivet/harvester/datamodel/DerbySpecifics.java index 02288bbc18..816ad125a2 100644 --- a/harvester/harvester-core/src/main/java/dk/netarkivet/harvester/datamodel/DerbySpecifics.java +++ b/harvester/harvester-core/src/main/java/dk/netarkivet/harvester/datamodel/DerbySpecifics.java @@ -390,10 +390,23 @@ protected void migrateExtendedFieldTableValueV1toV2() { HarvestDBConnection.updateTable("extendedfieldvalue", 2, sqlStatements); } - @Override protected void migrateOrderTemplatesTablev1tov2() { + @Override + protected void migrateOrderTemplatesTablev1tov2() { String tableName = HarvesterDatabaseTables.ORDERTEMPLATES.getTablename(); String[] sqlStatements = {"ALTER TABLE " + tableName + " ADD COLUMN isActive BOOLEAN NOT NULL DEFAULT TRUE"}; HarvestDBConnection.updateTable(tableName, 2, sqlStatements); } + @Override + public void createEavTypeAttributeTable(int toVersion) { + String tableName = HarvesterDatabaseTables.EAVTYPEATTRIBUTE.getTablename(); + HarvestDBConnection.executeSql("derby", tableName, 1 ); + } + + @Override + public void createEavAttributeTable(int toVersion) { + String tableName = HarvesterDatabaseTables.EAVATTRIBUTE.getTablename(); + HarvestDBConnection.executeSql("derby", tableName, 1 ); + } + } diff --git a/harvester/harvester-core/src/main/java/dk/netarkivet/harvester/datamodel/HarvestDBConnection.java b/harvester/harvester-core/src/main/java/dk/netarkivet/harvester/datamodel/HarvestDBConnection.java index 5924a6a5d7..8c0b410cfd 100644 --- a/harvester/harvester-core/src/main/java/dk/netarkivet/harvester/datamodel/HarvestDBConnection.java +++ b/harvester/harvester-core/src/main/java/dk/netarkivet/harvester/datamodel/HarvestDBConnection.java @@ -24,12 +24,17 @@ package dk.netarkivet.harvester.datamodel; import java.beans.PropertyVetoException; +import java.io.IOException; +import java.io.InputStream; import java.sql.Connection; import java.sql.SQLException; +import java.util.List; +import java.util.Map; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.antiaction.raptor.sql.ExecuteSqlFile; import com.mchange.v2.c3p0.ComboPooledDataSource; import dk.netarkivet.common.CommonSettings; @@ -117,7 +122,6 @@ public static synchronized Connection get() { * @throws IOFailure in case of problems in interacting with the database */ protected static void updateTable(final String table, final int newVersion, final String... updates) { - Connection c = get(); updateTable(c, table, newVersion, updates); } @@ -143,6 +147,27 @@ public static void updateTable(Connection c, final String table, final int newVe } } + protected static void updateTableVersion(final String table, final int newVersion, final String... updates) { + Connection c = get(); + updateTableVersion(c, table, newVersion); + } + + public static void updateTableVersion(Connection c, final String table, final int newVersion) { + log.info("Updating table '{}' to version {}", table, newVersion); + String updateSchemaversionSql = null; + if (newVersion == 1) { + updateSchemaversionSql = "INSERT INTO schemaversions(tablename, version) VALUES ('" + table + "', 1)"; + } else { + updateSchemaversionSql = "UPDATE schemaversions SET version = " + newVersion + " WHERE tablename = '" + + table + "'"; + } + try { + DBUtils.executeSQL(c, updateSchemaversionSql); + } finally { + release(c); + } + } + /** * Method for retrieving the url for the harvest definition database. This url will be constructed from the * base-url, the machine, the port and the directory. If the database is internal, then only the base-url should @@ -279,4 +304,35 @@ private static void initDataSource(DBSpecifics dbSpec, String jdbcUrl) throws SQ dataSource.getPreferredTestQuery(), dataSource.isTestConnectionOnCheckin()); } } + + public static void executeSql(String dbm, String tableName, int version) { + Connection conn = HarvestDBConnection.get(); + executeSql(conn, dbm, tableName, version); + HarvestDBConnection.release(conn); + conn = null; + } + + public static void executeSql(Connection conn, String dbm, String tableName, int version) { + InputStream in = DerbySpecifics.class.getClassLoader().getResourceAsStream("sql-migration/" + dbm +"/" + tableName + "." + version + ".sql"); + try { + List> statements = ExecuteSqlFile.splitSql(in, "UTF-8", 8192); + in.close(); + in = null; + ExecuteSqlFile.executeStatements(conn, statements); + HarvestDBConnection.updateTable(conn, tableName, version); + } catch (IOException e) { + e.printStackTrace(); + } catch (SQLException e) { + e.printStackTrace(); + } finally { + if (in != null) { + try { + in.close(); + } catch (IOException e) { + } + in = null; + } + } + } + } diff --git a/harvester/harvester-core/src/main/java/dk/netarkivet/harvester/datamodel/HarvesterDatabaseTables.java b/harvester/harvester-core/src/main/java/dk/netarkivet/harvester/datamodel/HarvesterDatabaseTables.java index 1456e736fd..1d87ef193a 100644 --- a/harvester/harvester-core/src/main/java/dk/netarkivet/harvester/datamodel/HarvesterDatabaseTables.java +++ b/harvester/harvester-core/src/main/java/dk/netarkivet/harvester/datamodel/HarvesterDatabaseTables.java @@ -409,6 +409,36 @@ public int getRequiredVersion() { return REQUIRED_VERSION; } + @Override + public String getTablename() { + return NAME; + } + }, + /** Harvest channels. */ + EAVTYPEATTRIBUTE { + static final String NAME = "eav_type_attribute"; + static final int REQUIRED_VERSION = 1; + + @Override + public int getRequiredVersion() { + return REQUIRED_VERSION; + } + + @Override + public String getTablename() { + return NAME; + } + }, + /** Harvest channels. */ + EAVATTRIBUTE { + static final String NAME = "eav_attribute"; + static final int REQUIRED_VERSION = 1; + + @Override + public int getRequiredVersion() { + return REQUIRED_VERSION; + } + @Override public String getTablename() { return NAME; diff --git a/harvester/harvester-core/src/main/java/dk/netarkivet/harvester/datamodel/MySQLSpecifics.java b/harvester/harvester-core/src/main/java/dk/netarkivet/harvester/datamodel/MySQLSpecifics.java index 660ac3edce..ae53536cd4 100644 --- a/harvester/harvester-core/src/main/java/dk/netarkivet/harvester/datamodel/MySQLSpecifics.java +++ b/harvester/harvester-core/src/main/java/dk/netarkivet/harvester/datamodel/MySQLSpecifics.java @@ -410,4 +410,16 @@ protected void migrateExtendedFieldTableValueV1toV2() { HarvestDBConnection.updateTable(tableName, 2, sqlStatements); } + @Override + public void createEavTypeAttributeTable(int toVersion) { + String tableName = HarvesterDatabaseTables.EAVTYPEATTRIBUTE.getTablename(); + HarvestDBConnection.executeSql("mysql", tableName, 1 ); + } + + @Override + public void createEavAttributeTable(int toVersion) { + String tableName = HarvesterDatabaseTables.EAVATTRIBUTE.getTablename(); + HarvestDBConnection.executeSql("mysql", tableName, 1 ); + } + } diff --git a/harvester/harvester-core/src/main/java/dk/netarkivet/harvester/datamodel/PostgreSQLSpecifics.java b/harvester/harvester-core/src/main/java/dk/netarkivet/harvester/datamodel/PostgreSQLSpecifics.java index 8047f83afb..4a721f54e7 100644 --- a/harvester/harvester-core/src/main/java/dk/netarkivet/harvester/datamodel/PostgreSQLSpecifics.java +++ b/harvester/harvester-core/src/main/java/dk/netarkivet/harvester/datamodel/PostgreSQLSpecifics.java @@ -345,11 +345,23 @@ protected void migrateExtendedFieldTableValueV1toV2() { HarvestDBConnection.updateTable("extendedfieldvalue", 2, sqlStatements); } - - @Override protected void migrateOrderTemplatesTablev1tov2() { + @Override + protected void migrateOrderTemplatesTablev1tov2() { final String tablename = HarvesterDatabaseTables.ORDERTEMPLATES.getTablename(); String[] sqlStatements = {"ALTER TABLE " + tablename + " ADD COLUMN isActive BOOL NOT NULL DEFAULT TRUE"}; HarvestDBConnection.updateTable(tablename, 2, sqlStatements); } + @Override + public void createEavTypeAttributeTable(int toVersion) { + String tableName = HarvesterDatabaseTables.EAVTYPEATTRIBUTE.getTablename(); + HarvestDBConnection.executeSql("postgresql", tableName, 1 ); + } + + @Override + public void createEavAttributeTable(int toVersion) { + String tableName = HarvesterDatabaseTables.EAVATTRIBUTE.getTablename(); + HarvestDBConnection.executeSql("postgresql", tableName, 1 ); + } + } diff --git a/harvester/harvester-core/src/main/java/dk/netarkivet/harvester/webinterface/SnapshotHarvestDefinition.java b/harvester/harvester-core/src/main/java/dk/netarkivet/harvester/webinterface/SnapshotHarvestDefinition.java index 96927f606a..b38442696c 100644 --- a/harvester/harvester-core/src/main/java/dk/netarkivet/harvester/webinterface/SnapshotHarvestDefinition.java +++ b/harvester/harvester-core/src/main/java/dk/netarkivet/harvester/webinterface/SnapshotHarvestDefinition.java @@ -200,9 +200,12 @@ public void processRequest(PageContext context, I18n i18n) { // EAV try { - long entity_id = hd.getOid(); - EAV eav = eavDAOProvider.get(); - List attributesAndTypes = eav.getAttributesAndTypes(EAV.SNAPSHOT_TREE_ID, (int)entity_id); + Long entity_id = hd.getOid(); + if (entity_id == null) { + entity_id = 0L; + } + EAV eav = eavDAOProvider.get(); + List attributesAndTypes = eav.getAttributesAndTypes(EAV.SNAPSHOT_TREE_ID, (int)((long)entity_id)); AttributeAndType attributeAndType; AttributeTypeBase attributeType; AttributeBase attribute; @@ -212,7 +215,7 @@ public void processRequest(PageContext context, I18n i18n) { attribute = attributeAndType.attribute; if (attribute == null) { attribute = attributeType.instanceOf(); - attribute.entity_id = (int)entity_id; + attribute.entity_id = (int)((long)entity_id); } switch (attributeType.viewtype) { case 1: diff --git a/pom.xml b/pom.xml index 75af977077..a6408bb0b1 100644 --- a/pom.xml +++ b/pom.xml @@ -299,6 +299,18 @@ jwat-warc ${jwat.version} + + + com.antiaction + raptor-dbl + 0.1.0-RC2 + + + * + * + + +