Skip to content

Option to migrate SQL Server hard tables to PostgreSQL #907

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 8 commits into from
Aug 1, 2025
Merged

Option to migrate SQL Server hard tables to PostgreSQL #907

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
7505542
Option to migrate SQL Server hard tables to PostgreSQL
labkey-adam Jul 23, 2025
c48cdc2
Register handlers for schema-specific functionality
labkey-adam Jul 25, 2025
b5081f2
Remove unnecessary override
labkey-adam Jul 26, 2025
50489b8
Support skip annotations in comments for convenience
labkey-adam Jul 26, 2025
dd043dc
Drop unused, problematic Deleted columns in prot schema
labkey-adam Jul 27, 2025
39f5eed
Move FK to correct table
labkey-adam Jul 28, 2025
e30b841
Don't redirect to a string
labkey-adam Jul 28, 2025
4acb171
Merge remote-tracking branch 'origin/develop' into fb_migrate_ss_to_pg
labkey-adam Jul 30, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 3 additions & 4 deletions flow/src/org/labkey/flow/controllers/FlowController.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@
import org.labkey.api.util.FileUtil;
import org.labkey.api.util.JobRunner;
import org.labkey.api.util.TestContext;
import org.labkey.api.util.URLHelper;
import org.labkey.api.view.ActionURL;
import org.labkey.api.view.HttpView;
import org.labkey.api.view.JspView;
Expand All @@ -72,8 +73,6 @@
import org.springframework.validation.Errors;
import org.springframework.web.servlet.ModelAndView;

import java.net.URI;

@Marshal(Marshaller.Jackson)
public class FlowController extends BaseFlowController
{
Expand Down Expand Up @@ -367,8 +366,8 @@ public class SavePreferencesAction extends SimpleViewAction<Object>
public ModelAndView getView(Object o, BindException errors) throws Exception
{
FlowPreference.update(getRequest());
URI uri = new URI(getRequest().getContextPath() + "/_.gif");
return HttpView.redirect(uri.toString());
URLHelper url = new URLHelper(getRequest().getContextPath() + "/_.gif");
return HttpView.redirect(url, false);
}

@Override
Expand Down
10 changes: 10 additions & 0 deletions protein/resources/schemas/dbscripts/postgresql/prot-0.000-21.000.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,7 @@ CREATE TABLE prot.InfoSources

/*** Initializations */

-- @SkipOnEmptySchemasBegin
INSERT INTO prot.InfoSources (Name, Url, InsertDate) VALUES ('Genbank', 'http://www.ncbi.nlm.nih.gov/entrez/query.fcgi?db=protein&cmd=search&term={}', '2005-03-04 12:08:10');
INSERT INTO prot.InfoSources (Name, Url, InsertDate) VALUES ('NiceProt', 'http://au.expasy.org/cgi-bin/niceprot.pl?{}', '2005-03-04 12:08:10');
INSERT INTO prot.InfoSources (Name, Url, InsertDate) VALUES ('GeneCards', 'http://www.genecards.org/cgi-bin/carddisp?{}&alias=yes', '2005-03-04 12:08:10');
Expand All @@ -80,6 +81,7 @@ UPDATE prot.InfoSources SET Url = 'http://www.genecards.org/cgi-bin/carddisp.pl?
UPDATE prot.InfoSources SET URL = 'http://www.uniprot.org/uniprot/{}' WHERE Name = 'NiceProt';
UPDATE prot.InfoSources SET URL = 'http://www.ncbi.nlm.nih.gov/protein/{}' WHERE Name = 'Genbank';
UPDATE prot.infosources SET url = 'http://amigo.geneontology.org/amigo/term/{}' WHERE Name = 'GO';
-- @SkipOnEmptySchemasEnd

/****** AnnotationTypes */
CREATE TABLE prot.AnnotationTypes
Expand All @@ -97,13 +99,15 @@ CREATE TABLE prot.AnnotationTypes
);
CREATE UNIQUE INDEX UQ_ProtAnnotationTypes ON prot.AnnotationTypes(Name);

-- @SkipOnEmptySchemasBegin
INSERT INTO prot.AnnotationTypes (Name,SourceId,EntryDate) VALUES ('GO_F',5,'2005-03-04 11:37:15');
INSERT INTO prot.AnnotationTypes (Name,SourceId,EntryDate) VALUES ('GO_P',5,'2005-03-04 11:37:15');
INSERT INTO prot.AnnotationTypes (Name,EntryDate) VALUES ('keyword','2005-03-04 11:37:15');
INSERT INTO prot.AnnotationTypes (Name,EntryDate) VALUES ('feature','2005-03-04 11:37:15');
INSERT INTO prot.AnnotationTypes (Name,SourceId,EntryDate) VALUES ('GO_C',5,'2005-03-04 11:38:13');
INSERT INTO prot.AnnotationTypes (Name,EntryDate) VALUES ('FullOrganismName',now());
INSERT INTO prot.AnnotationTypes (Name,EntryDate) VALUES ('LookupString',now());
-- @SkipOnEmptySchemasEnd

CREATE INDEX IX_AnnotationTypes_SourceId ON prot.annotationtypes(SourceId);

Expand All @@ -124,6 +128,7 @@ CREATE TABLE prot.IdentTypes
CREATE UNIQUE INDEX UQ_ProtIdentTypes ON prot.IdentTypes(Name);
CREATE INDEX IX_IdentTypes_cannonicalsourceid ON prot.IdentTypes(cannonicalsourceid);

-- @SkipOnEmptySchemasBegin
INSERT INTO prot.IdentTypes (Name,CannonicalSourceId,EntryDate) VALUES ('Genbank',1,'2005-03-04 11:37:14');
INSERT INTO prot.IdentTypes (Name,CannonicalSourceId,EntryDate) VALUES ('SwissProt',2,'2005-03-04 11:37:14');
INSERT INTO prot.IdentTypes (Name,CannonicalSourceId,EntryDate) VALUES ('GeneName',3,'2005-03-04 11:37:14');
Expand Down Expand Up @@ -193,6 +198,7 @@ INSERT INTO prot.IdentTypes (Name,EntryDate) VALUES ('PhosSite','2005-03-04 12:4
INSERT INTO prot.IdentTypes (Name,EntryDate) VALUES ('REBASE','2005-03-04 13:25:29');
INSERT INTO prot.IdentTypes (Name,EntryDate) VALUES ('Maize-2DPAGE','2005-03-04 15:10:53');
INSERT INTO prot.IdentTypes (Name,EntryDate) VALUES ('HIV','2005-03-04 22:13:40');
-- @SkipOnEmptySchemasEnd

/****** Organisms */
CREATE TABLE prot.Organisms
Expand All @@ -209,7 +215,9 @@ CREATE TABLE prot.Organisms
);
CREATE UNIQUE INDEX UQ_ProtOrganisms_Genus_Species ON prot.Organisms(Genus, Species);

-- @SkipOnEmptySchemasBegin
INSERT INTO prot.Organisms (CommonName,Genus,Species,Comments) VALUES ('Unknown organism','Unknown','unknown','Organism is unknown');
-- @SkipOnEmptySchemasEnd

CREATE INDEX IX_Organisms_IdentId ON prot.Organisms(IdentId);

Expand Down Expand Up @@ -497,6 +505,7 @@ CREATE FUNCTION prot.drop_go_indexes() RETURNS void AS $$
END;
$$ LANGUAGE plpgsql;

-- @SkipOnEmptySchemasBegin
-- add most common ncbi Taxonomy id's

CREATE TEMPORARY TABLE idents
Expand Down Expand Up @@ -568,6 +577,7 @@ UPDATE prot.Organisms
WHERE i.OrgId = Organisms.OrgId;

DROP TABLE idents;
-- @SkipOnEmptySchemasEnd

CREATE TABLE prot.FastaSequences
(
Expand Down
8 changes: 8 additions & 0 deletions protein/resources/schemas/dbscripts/postgresql/prot-25.000-25.001.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
-- These columns are unused and, due to data type differences, are problematic for SQL Server -> PostgreSQL migration
ALTER TABLE prot.InfoSources DROP COLUMN Deleted;
ALTER TABLE prot.AnnotationTypes DROP COLUMN Deleted;
ALTER TABLE prot.IdentTypes DROP COLUMN Deleted;
ALTER TABLE prot.Organisms DROP COLUMN Deleted;
ALTER TABLE prot.Sequences DROP COLUMN Deleted;
ALTER TABLE prot.Identifiers DROP COLUMN Deleted;
ALTER TABLE prot.Annotations DROP COLUMN Deleted;
8 changes: 8 additions & 0 deletions protein/resources/schemas/dbscripts/sqlserver/prot-25.000-25.001.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
-- These columns are unused and, due to data type differences, are problematic for SQL Server -> PostgreSQL migration
EXEC core.fn_dropifexists 'InfoSources', 'prot', 'COLUMN', 'Deleted';
EXEC core.fn_dropifexists 'AnnotationTypes', 'prot', 'COLUMN', 'Deleted';
EXEC core.fn_dropifexists 'IdentTypes', 'prot', 'COLUMN', 'Deleted';
EXEC core.fn_dropifexists 'Organisms', 'prot', 'COLUMN', 'Deleted';
EXEC core.fn_dropifexists 'Sequences', 'prot', 'COLUMN', 'Deleted';
EXEC core.fn_dropifexists 'Identifiers', 'prot', 'COLUMN', 'Deleted';
EXEC core.fn_dropifexists 'Annotations', 'prot', 'COLUMN', 'Deleted';
3 changes: 3 additions & 0 deletions protein/resources/schemas/dbscripts/sqlserver/prot-25.001-25.002.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
-- Move foreign key to the correct table. Issue 53523.
ALTER TABLE prot.Organisms DROP CONSTRAINT FK_ProtSequences_ProtOrganisms;
ALTER TABLE prot.Sequences ADD CONSTRAINT FK_ProtSequences_ProtOrganisms FOREIGN KEY (OrgId) REFERENCES prot.Organisms (OrgId);
17 changes: 0 additions & 17 deletions protein/resources/schemas/prot.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,9 +38,6 @@
</column>
<column columnName="StartPos"/>
<column columnName="EndPos"/>
<column columnName="Deleted">
<isHidden>true</isHidden>
</column>
</columns>
</table>
<table tableName="AnnotationTypes" tableDbType="TABLE">
Expand All @@ -53,7 +50,6 @@
<column columnName="Description"/>
<column columnName="EntryDate"/>
<column columnName="ModDate"/>
<column columnName="Deleted"/>
</columns>
</table>
<table tableName="AnnotInsertions" tableDbType="TABLE">
Expand Down Expand Up @@ -135,9 +131,6 @@
<column columnName="SourceVersion">
<isHidden>true</isHidden>
</column>
<column columnName="Deleted">
<isHidden>true</isHidden>
</column>
</columns>
</table>
<table tableName="IdentTypes" tableDbType="TABLE">
Expand All @@ -151,9 +144,6 @@
</column>
<column columnName="EntryDate"/>
<column columnName="Description"/>
<column columnName="Deleted">
<isHidden>true</isHidden>
</column>
</columns>
</table>
<table tableName="InfoSources" tableDbType="TABLE">
Expand All @@ -169,7 +159,6 @@
<column columnName="LastUpdate"/>
<column columnName="InsertDate"/>
<column columnName="ModDate"/>
<column columnName="Deleted"/>
</columns>
</table>
<table tableName="Organisms" tableDbType="TABLE">
Expand All @@ -185,9 +174,6 @@
<column columnName="IdentId">
<isHidden>true</isHidden>
</column>
<column columnName="Deleted">
<isHidden>true</isHidden>
</column>
</columns>
</table>
<table tableName="Sequences" tableDbType="TABLE">
Expand Down Expand Up @@ -229,9 +215,6 @@
<column columnName="BestName"/>
<column columnName="BestGeneName"/>
<column columnName="Length"/>
<column columnName="Deleted">
<isHidden>true</isHidden>
</column>
</columns>
<description>Contains a row per protein known to this server installation.</description>
</table>
Expand Down
20 changes: 19 additions & 1 deletion protein/src/org/labkey/protein/ProteinModule.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,9 +20,13 @@
import org.jetbrains.annotations.Nullable;
import org.labkey.api.data.Container;
import org.labkey.api.data.ContainerManager;
import org.labkey.api.data.DbSchema;
import org.labkey.api.data.SqlExecutor;
import org.labkey.api.data.TableSelector;
import org.labkey.api.files.FileContentService;
import org.labkey.api.files.TableUpdaterFileListener;
import org.labkey.api.module.DatabaseMigration;
import org.labkey.api.module.DatabaseMigration.DefaultMigrationHandler;
import org.labkey.api.module.DefaultModule;
import org.labkey.api.module.ModuleContext;
import org.labkey.api.pipeline.PipelineService;
Expand Down Expand Up @@ -62,7 +66,7 @@ public String getName()
@Override
public @Nullable Double getSchemaVersion()
{
return 25.000;
return 25.001;
}

@Override
Expand Down Expand Up @@ -125,6 +129,20 @@ public void doStartup(ModuleContext moduleContext)
}

ProteinService.get().registerProteinSearchView(new ProteinSearchViewProvider());
DatabaseMigration.registerHandler(ProteinSchema.getSchema(), new DefaultMigrationHandler()
{
@Override
public void beforeSchema(DbSchema targetSchema)
{
new SqlExecutor(targetSchema).execute("ALTER TABLE prot.Organisms DROP CONSTRAINT FK_ProtOrganisms_ProtIdentifiers");
}

@Override
public void afterSchema(DbSchema targetSchema)
{
new SqlExecutor(targetSchema).execute("ALTER TABLE prot.Organisms ADD CONSTRAINT FK_ProtOrganisms_ProtIdentifiers FOREIGN KEY (IdentId) REFERENCES prot.Identifiers (IdentId)");
}
});
}

@Override
Expand Down