Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Fixes #14004: Wasted disk space due to inefficient columns ordering #2099

Closed
wants to merge 1 commit into from
Closed

Fixes #14004: Wasted disk space due to inefficient columns ordering #2099

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 8 additions & 8 deletions ...-core/src/main/resources/Migration/dbMigration-4.1.x-4.1.12-add-compliancelevel-table.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,10 +36,7 @@
*/

CREATE TABLE nodecompliancelevels (
nodeId text NOT NULL CHECK (nodeId <> '')
, runTimestamp timestamp with time zone NOT NULL
, ruleId text NOT NULL CHECK (nodeId <> '')
, directiveId text NOT NULL CHECK (nodeId <> '')
runTimestamp timestamp with time zone NOT NULL
, pending int DEFAULT 0
, success int DEFAULT 0
, repaired int DEFAULT 0
Expand All @@ -54,10 +51,13 @@ CREATE TABLE nodecompliancelevels (
, nonCompliant int DEFAULT 0
, auditError int DEFAULT 0
, badPolicyMode int DEFAULT 0
, nodeId text NOT NULL CHECK (nodeId <> '')
, ruleId text NOT NULL CHECK (nodeId <> '')
, directiveId text NOT NULL CHECK (nodeId <> '')
, PRIMARY KEY (nodeId, runTimestamp, ruleId, directiveId)
);

CREATE INDEX nodecompliancelevels_nodeId ON nodecompliancelevels (nodeId);
CREATE INDEX nodecompliancelevels_ruleId ON nodecompliancelevels (nodeId);
CREATE INDEX nodecompliancelevels_directiveId ON nodecompliancelevels (nodeId);
CREATE INDEX nodecompliancelevels_runTimestamp ON nodecompliancelevels (runTimestamp);
CREATE INDEX nodecompliancelevels_nodeId_idx ON nodecompliancelevels (nodeId);
CREATE INDEX nodecompliancelevels_ruleId_idx ON nodecompliancelevels (nodeId);
CREATE INDEX nodecompliancelevels_directiveId_idx ON nodecompliancelevels (nodeId);
CREATE INDEX nodecompliancelevels_runTimestamp_idx ON nodecompliancelevels (runTimestamp);
71 changes: 30 additions & 41 deletions rudder-core/src/main/resources/reportsSchema.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -64,15 +64,15 @@ ALTER database rudder SET standard_conforming_strings=true;
CREATE SEQUENCE serial START 101;

CREATE TABLE RudderSysEvents (
id bigint PRIMARY KEY default nextval('serial')
, executionDate timestamp with time zone NOT NULL
executionDate timestamp with time zone NOT NULL
, executionTimeStamp timestamp with time zone NOT NULL
, id bigint PRIMARY KEY default nextval('serial')
, serial integer NOT NULL
, nodeId text NOT NULL CHECK (nodeId <> '')
, directiveId text NOT NULL CHECK (directiveId <> '')
, ruleId text NOT NULL CHECK (ruleId <> '')
, serial integer NOT NULL
, component text NOT NULL CHECK (component <> '')
, keyValue text
, executionTimeStamp timestamp with time zone NOT NULL
, eventType text
, policy text
, msg text
Expand All @@ -93,15 +93,15 @@ CREATE INDEX changes_executionTimeStamp_idx ON RudderSysEvents (executionTimeSta
* The table used to store archived agent execution reports.
*/
CREATE TABLE ArchivedRudderSysEvents (
id bigint PRIMARY KEY
, executionDate timestamp with time zone NOT NULL
executionDate timestamp with time zone NOT NULL
, executionTimeStamp timestamp with time zone NOT NULL
, id bigint PRIMARY KEY
, serial integer NOT NULL
, nodeId text NOT NULL CHECK (nodeId <> '')
, directiveId text NOT NULL CHECK (directiveId <> '')
, ruleId text NOT NULL CHECK (ruleId <> '')
, serial integer NOT NULL
, component text NOT NULL CHECK (component <> '')
, keyValue text
, executionTimeStamp timestamp with time zone NOT NULL
, eventType text
, policy text
, msg text
Expand All @@ -116,30 +116,18 @@ CREATE INDEX executionTimeStamp_archived_idx ON ArchivedRudderSysEvents (executi
* or not.
*/
CREATE TABLE ReportsExecution (
nodeId text NOT NULL
, date timestamp with time zone NOT NULL
date timestamp with time zone NOT NULL
, insertionId bigint
, complete boolean NOT NULL
, nodeId text NOT NULL
, nodeConfigId text
, insertionId bigint
, PRIMARY KEY(nodeId, date)
);

CREATE INDEX reportsexecution_date_idx ON ReportsExecution (date);
CREATE INDEX reportsexecution_insertionid_idx ON ReportsExecution (insertionId);
CREATE INDEX reportsexecution_nodeid_nodeconfigid_idx ON ReportsExecution (nodeId, nodeConfigId);

/*
* Store the archived agent execution times for each nodes.
*/
CREATE TABLE ArchivedReportsExecution (
nodeId text NOT NULL
, date timestamp with time zone NOT NULL
, complete boolean NOT NULL
, nodeConfigId text
, insertionId bigint
, PRIMARY KEY(nodeId, date)
);

/*
*************************************************************************************
* The following tables store what Rudder expects from agent.
Expand All @@ -161,11 +149,12 @@ CREATE TABLE nodes_info (

-- Create the table for the node configuration
CREATE TABLE nodeConfigurations (
nodeId text NOT NULL CHECK (nodeId <> '')
, nodeConfigId text NOT NULL CHECK (nodeConfigId <> '')
, beginDate timestamp with time zone NOT NULL
beginDate timestamp with time zone NOT NULL
, endDate timestamp with time zone

, nodeId text NOT NULL CHECK (nodeId <> '')
, nodeConfigId text NOT NULL CHECK (nodeConfigId <> '')

-- here, I'm using text but with valid JSON in it, because for now we can't impose postgres > 9.2,
-- and interesting function are on 9.3/9.4. we will be able to migrate with:
-- ALTER TABLE table1 ALTER COLUMN col1 TYPE JSON USING col1::JSON;
Expand All @@ -189,10 +178,10 @@ CREATE INDEX nodeConfigurations_nodeConfigId ON nodeConfigurations (nodeConfigId

-- Create the table for the archived node configurations
CREATE TABLE archivedNodeConfigurations (
nodeId text NOT NULL CHECK (nodeId <> '')
, nodeConfigId text NOT NULL CHECK (nodeConfigId <> '')
, beginDate timestamp with time zone NOT NULL
beginDate timestamp with time zone NOT NULL
, endDate timestamp with time zone
, nodeId text NOT NULL CHECK (nodeId <> '')
, nodeConfigId text NOT NULL CHECK (nodeConfigId <> '')
, configuration text NOT NULL CHECK (configuration <> '' )
, PRIMARY KEY (nodeId, nodeConfigId, beginDate)
);
Expand All @@ -210,14 +199,14 @@ CREATE TABLE archivedNodeConfigurations (

-- Create the table for the node compliance
CREATE TABLE nodeCompliance (
nodeId text NOT NULL CHECK (nodeId <> '')
, runTimestamp timestamp with time zone NOT NULL
runTimestamp timestamp with time zone NOT NULL

-- endOfList is the date until which the compliance information
-- are relevant. After that date/time, the node must have sent
-- a more recent run, this one is not valide anymore.
, endOfLife timestamp with time zone

, nodeId text NOT NULL CHECK (nodeId <> '')
-- all information about the run and what lead to that compliance:
-- the run config version, the awaited config version, etc
-- It's JSON (but in a string, cf explanation in nodeConfigurations table)
Expand Down Expand Up @@ -250,9 +239,9 @@ CREATE INDEX nodeCompliance_endOfLife ON nodeCompliance (endOfLife);

-- Create the table for the archived node compliance
CREATE TABLE archivedNodeCompliance (
nodeId text NOT NULL CHECK (nodeId <> '')
, runTimestamp timestamp with time zone NOT NULL
runTimestamp timestamp with time zone NOT NULL
, endOfLife timestamp with time zone
, nodeId text NOT NULL CHECK (nodeId <> '')
, runAnalysis text NOT NULL CHECK (runAnalysis <> '' )
, summary text NOT NULL CHECK (summary <> '' )
, details text NOT NULL CHECK (details <> '' )
Expand All @@ -265,10 +254,7 @@ CREATE TABLE archivedNodeCompliance (
-- but with a much more reasonable space until all our supported server versions
-- have at least PostgreSQL 9.4.
CREATE TABLE nodecompliancelevels (
nodeId text NOT NULL CHECK (nodeId <> '')
, runTimestamp timestamp with time zone NOT NULL
, ruleId text NOT NULL CHECK (nodeId <> '')
, directiveId text NOT NULL CHECK (nodeId <> '')
runTimestamp timestamp with time zone NOT NULL
, pending int DEFAULT 0
, success int DEFAULT 0
, repaired int DEFAULT 0
Expand All @@ -283,13 +269,16 @@ CREATE TABLE nodecompliancelevels (
, nonCompliant int DEFAULT 0
, auditError int DEFAULT 0
, badPolicyMode int DEFAULT 0
, nodeId text NOT NULL CHECK (nodeId <> '')
, ruleId text NOT NULL CHECK (nodeId <> '')
, directiveId text NOT NULL CHECK (nodeId <> '')
, PRIMARY KEY (nodeId, runTimestamp, ruleId, directiveId)
);

CREATE INDEX nodecompliancelevels_nodeId ON nodecompliancelevels (nodeId);
CREATE INDEX nodecompliancelevels_ruleId ON nodecompliancelevels (nodeId);
CREATE INDEX nodecompliancelevels_directiveId ON nodecompliancelevels (nodeId);
CREATE INDEX nodecompliancelevels_runTimestamp ON nodecompliancelevels (runTimestamp);
CREATE INDEX nodecompliancelevels_nodeId_idx ON nodecompliancelevels (nodeId);
CREATE INDEX nodecompliancelevels_ruleId_idx ON nodecompliancelevels (ruleId);
CREATE INDEX nodecompliancelevels_directiveId_idx ON nodecompliancelevels (directiveId);
CREATE INDEX nodecompliancelevels_runTimestamp_idx ON nodecompliancelevels (runTimestamp);



Expand Down
4 changes: 2 additions & 2 deletions ...e/src/main/scala/com/normation/rudder/repository/jdbc/ExpectedReportsJdbcRepository.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -402,8 +402,8 @@ class UpdateExpectedReportsJdbcRepository(
val dateAt_0000 = date.toString("yyyy-MM-dd")
val copy = s"""
insert into archivednodecompliance
(nodeid, runtimestamp, endoflife, runanalysis, summary, details)
(select nodeid, runtimestamp, endoflife, runanalysis, summary, details
(runtimestamp, endoflife, nodeid, runanalysis, summary, details)
(select runtimestamp, endoflife, nodeid, runanalysis, summary, details
from nodecompliance
where coalesce(runtimestamp, '${dateAt_0000}') < '${dateAt_0000}'
)
Expand Down