Skip to content
Permalink
Browse files

Delete entities associated with a job

When a job completes, a lot of entities are left behind in the DB. These
persisted entities are copies of the config at the time the job was
scheduled, and is not used once the job completes.

For jobs that are already completed (as per DB), and for those jobs that
will complete from this point onwards:

* delete environment variables that are not trigger-with-options
* delete all artifact plans that are of type `file`. Artifact plans of
  type `unit` are retained for shine.
* delete all artifact properties generators
* delete all (agent) resources associated with a job
* delete all elastic agent metatada

Since some of the data in the DB will eventually go missing, the job
feed api will break. We will no longer render the environment variables,
resources and build artifacts in the job xml api.
  • Loading branch information...
ketan committed Dec 26, 2017
1 parent cba9fd7 commit 23eb465f0b366378c526c70642c3d0748ce47e2e
Showing with 589 additions and 150 deletions.
  1. +16 −4 domain/src/main/java/com/thoughtworks/go/domain/DefaultJobPlan.java
  2. +22 −0 domain/src/main/java/com/thoughtworks/go/domain/EnvironmentVariableType.java
  3. +2 −0 domain/src/main/java/com/thoughtworks/go/domain/EnvironmentVariables.java
  4. +6 −3 domain/src/main/java/com/thoughtworks/go/domain/JobPlan.java
  5. +2 −2 domain/src/test/java/com/thoughtworks/go/domain/EnvironmentVariablesTest.java
  6. +14 −0 domain/src/test/java/com/thoughtworks/go/helper/JobInstanceMother.java
  7. +31 −0 server/db/migrate/h2deltas/1801003_remove_environment_variables_for_completed_jobs.sql
  8. +31 −0 server/db/migrate/h2deltas/1801004_remove_artifact_plans_for_completed_jobs.sql
  9. +28 −0 server/db/migrate/h2deltas/1801005_remove_artifact_properties_generator_for_completed_jobs.sql
  10. +28 −0 server/db/migrate/h2deltas/1801006_remove_resources_for_completed_jobs.sql
  11. +29 −0 server/db/migrate/h2deltas/1801007_remove_job_agent_metadata_for_completed_jobs.sql
  12. +8 −6 server/src/main/java/com/thoughtworks/go/server/dao/EnvironmentVariableDao.java
  13. +16 −2 server/src/main/java/com/thoughtworks/go/server/dao/EnvironmentVariableSqlMapDao.java
  14. +3 −1 server/src/main/java/com/thoughtworks/go/server/dao/JobAgentMetadataDao.java
  15. +11 −1 server/src/main/java/com/thoughtworks/go/server/dao/JobAgentMetadataSqlMapDao.java
  16. +20 −4 server/src/main/java/com/thoughtworks/go/server/dao/JobInstanceSqlMapDao.java
  17. +2 −2 server/src/main/java/com/thoughtworks/go/server/dao/PipelineSqlMapDao.java
  18. +11 −16 server/src/main/java/com/thoughtworks/go/server/domain/xml/JobXmlViewModel.java
  19. +8 −4 server/src/main/java/com/thoughtworks/go/server/persistence/ArtifactPlanRepository.java
  20. +8 −4 ...r/src/main/java/com/thoughtworks/go/server/persistence/ArtifactPropertiesGeneratorRepository.java
  21. +10 −4 server/src/main/java/com/thoughtworks/go/server/persistence/ResourceRepository.java
  22. +1 −1 server/src/main/java/com/thoughtworks/go/server/service/ScheduleService.java
  23. +0 −67 server/src/test-fast/java/com/thoughtworks/go/server/domain/xml/JobXmlViewModelTest.java
  24. +23 −5 ...-integration/java/com/thoughtworks/go/server/dao/EnvironmentVariableSqlMapDaoIntegrationTest.java
  25. +76 −2 server/src/test-integration/java/com/thoughtworks/go/server/dao/JobInstanceSqlMapDaoTest.java
  26. +17 −2 ...ntegration/java/com/thoughtworks/go/server/persistence/ArtifactPlanRepositoryIntegrationTest.java
  27. +24 −4 .../com/thoughtworks/go/server/persistence/ArtifactPropertiesGeneratorRepositoryIntegrationTest.java
  28. +26 −4 ...st-integration/java/com/thoughtworks/go/server/persistence/ResourceRepositoryIntegrationTest.java
  29. +18 −0 .../src/test-integration/java/com/thoughtworks/go/server/service/ScheduleServiceIntegrationTest.java
  30. +98 −0 server/test/integration/com/thoughtworks/go/server/dao/JobAgentMetadataSqlMapDaoIntegrationTest.java
  31. +0 −12 server/webapp/WEB-INF/rails.new/spec/views/api/jobs/index_xml_spec.rb
@@ -1,5 +1,5 @@
/*
* Copyright 2016 ThoughtWorks, Inc.
* Copyright 2017 ThoughtWorks, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -22,6 +22,8 @@

import java.util.ArrayList;
import java.util.List;
import java.util.function.Predicate;
import java.util.stream.Collectors;

public class DefaultJobPlan implements JobPlan {

@@ -169,15 +171,15 @@ public EnvironmentVariables getVariables() {
}

public void setVariables(EnvironmentVariables variables) {
this.variables = variables;
this.variables = new EnvironmentVariables(variables);
}

public long getPipelineId() {
return pipelineId;
}

public void setTriggerVariables(EnvironmentVariables environmentVariables) {
triggerVariables = environmentVariables;
triggerVariables = new EnvironmentVariables(environmentVariables);
}

public boolean shouldFetchMaterials() {
@@ -215,7 +217,17 @@ public boolean requiresElasticAgent() {
}

public void setElasticProfile(ElasticProfile elasticProfile) {
this.elasticProfile = elasticProfile;
this.elasticProfile = new ElasticProfile(elasticProfile.getId(), elasticProfile.getPluginId(), elasticProfile);
}

@Override
public List<ArtifactPlan> getArtifactPlansOfType(final ArtifactType file) {
return getArtifactPlans().stream().filter(new Predicate<ArtifactPlan>() {
@Override
public boolean test(ArtifactPlan artifactPlan) {
return artifactPlan.getArtifactType() == file;
}
}).collect(Collectors.toList());
}

@Override
@@ -0,0 +1,22 @@
/*
* Copyright 2017 ThoughtWorks, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package com.thoughtworks.go.domain;

public enum EnvironmentVariableType {
Job,
Trigger
}
@@ -24,6 +24,8 @@

public class EnvironmentVariables extends BaseCollection<EnvironmentVariable> {

private static final String JOB = EnvironmentVariableType.Job.toString();

public EnvironmentVariables() {
}

@@ -1,5 +1,5 @@
/*
* Copyright 2016 ThoughtWorks, Inc.
* Copyright 2017 ThoughtWorks, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -52,8 +52,6 @@

long getPipelineId();

void setTriggerVariables(EnvironmentVariables environmentVariables);

boolean shouldFetchMaterials();

void setFetchMaterials(boolean fetchMaterials);
@@ -67,4 +65,9 @@
boolean requiresElasticAgent();

boolean assignedToAgent();

void setElasticProfile(ElasticProfile elasticProfile);

List<ArtifactPlan> getArtifactPlansOfType(final ArtifactType file);

}
@@ -25,8 +25,7 @@
import java.util.Arrays;

import static org.hamcrest.Matchers.contains;
import static org.hamcrest.Matchers.containsInAnyOrder;
import static org.hamcrest.Matchers.hasSize;
import static org.hamcrest.Matchers.*;
import static org.junit.Assert.assertThat;
import static org.mockito.Mockito.*;

@@ -88,4 +87,5 @@ public void addToIfExists_shouldAddEnvironmentVariableToEnvironmentVariableConte
verify(environmentVariableContext, times(0)).setProperty("foo", "bar", false);
verify(environmentVariableContext, times(1)).setProperty("baz", "car", true);
}

}
@@ -18,7 +18,11 @@

import com.thoughtworks.go.config.JobConfig;
import com.thoughtworks.go.config.ResourceConfigs;
import com.thoughtworks.go.config.elastic.ElasticProfile;
import com.thoughtworks.go.domain.*;
import com.thoughtworks.go.domain.config.ConfigurationKey;
import com.thoughtworks.go.domain.config.ConfigurationProperty;
import com.thoughtworks.go.domain.config.ConfigurationValue;
import org.joda.time.DateTime;

import java.util.ArrayList;
@@ -258,4 +262,14 @@ public static JobPlan createJobPlan(JobConfig jobConfig, JobIdentifier jobIdenti
return new DefaultJobPlan(new Resources(jobConfig.resourceConfigs()), artifactPlans, artifactPropertiesGenerators, -1,
jobIdentifier, null, environmentVariables, new EnvironmentVariables(), null);
}

public static JobPlan jobPlanWithAssociatedEntities(String jobName, long id, List<ArtifactPlan> artifactPlans, List<ArtifactPropertiesGenerator> artifactPropertiesGenerators) {
ConfigurationProperty configurationProperty = new ConfigurationProperty(new ConfigurationKey("image"), new ConfigurationValue("elastic-agent"));
ElasticProfile elasticProfile = new ElasticProfile("elastic", "plugin", configurationProperty);

EnvironmentVariables variables = new EnvironmentVariables();
variables.add("some_var", "blah");

return new DefaultJobPlan(new Resources(new Resource("foo"), new Resource("bar")), artifactPlans, artifactPropertiesGenerators, id, defaultJobIdentifier(jobName), null, variables, new EnvironmentVariables(), elasticProfile);
}
}
@@ -0,0 +1,31 @@
--
-- Copyright 2017 ThoughtWorks, Inc.
--
-- Licensed under the Apache License, Version 2.0 (the "License");
-- you may not use this file except in compliance with the License.
-- You may obtain a copy of the License at
--
-- http://www.apache.org/licenses/LICENSE-2.0
--
-- Unless required by applicable law or agreed to in writing, software
-- distributed under the License is distributed on an "AS IS" BASIS,
-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-- See the License for the specific language governing permissions and
-- limitations under the License.
--

DELETE FROM environmentVariables
WHERE environmentVariables.id IN (
SELECT environmentVariables.id
FROM environmentVariables
INNER JOIN builds
ON
builds.id = environmentVariables.entityId
WHERE
environmentVariables.entityType = 'Job'
AND
builds.state = 'Completed'
);

--//@UNDO

@@ -0,0 +1,31 @@
--
-- Copyright 2017 ThoughtWorks, Inc.
--
-- Licensed under the Apache License, Version 2.0 (the "License");
-- you may not use this file except in compliance with the License.
-- You may obtain a copy of the License at
--
-- http://www.apache.org/licenses/LICENSE-2.0
--
-- Unless required by applicable law or agreed to in writing, software
-- distributed under the License is distributed on an "AS IS" BASIS,
-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-- See the License for the specific language governing permissions and
-- limitations under the License.
--

DELETE FROM artifactPlans
WHERE artifactPlans.id IN (
SELECT artifactPlans.id
FROM artifactPlans
INNER JOIN builds
ON builds.id = artifactPlans.buildId
WHERE
artifactPlans.artifactType = 'file'
AND
builds.state = 'Completed'
);


--//@UNDO

@@ -0,0 +1,28 @@
--
-- Copyright 2017 ThoughtWorks, Inc.
--
-- Licensed under the Apache License, Version 2.0 (the "License");
-- you may not use this file except in compliance with the License.
-- You may obtain a copy of the License at
--
-- http://www.apache.org/licenses/LICENSE-2.0
--
-- Unless required by applicable law or agreed to in writing, software
-- distributed under the License is distributed on an "AS IS" BASIS,
-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-- See the License for the specific language governing permissions and
-- limitations under the License.
--

DELETE FROM artifactPropertiesGenerator
WHERE ID IN (
SELECT artifactPropertiesGenerator.id
FROM artifactPropertiesGenerator
INNER JOIN builds
ON builds.id = artifactPropertiesGenerator.jobId
WHERE
builds.state = 'Completed'
);

--//@UNDO

@@ -0,0 +1,28 @@
--
-- Copyright 2017 ThoughtWorks, Inc.
--
-- Licensed under the Apache License, Version 2.0 (the "License");
-- you may not use this file except in compliance with the License.
-- You may obtain a copy of the License at
--
-- http://www.apache.org/licenses /LICENSE-2.0
--
-- Unless required by applicable law or agreed to in writing, software
-- distributed under the License is distributed on an "AS IS" BASIS,
-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-- See the License for the specific language governing permissions and
-- limitations under the License.
--

DELETE FROM resources
WHERE resources.id IN (
SELECT resources.id
FROM resources
INNER JOIN builds
ON builds.id = resources.buildId
WHERE
builds.state = 'Completed'
);

--//@UNDO

@@ -0,0 +1,29 @@
--
-- Copyright 2017 ThoughtWorks, Inc.
--
-- Licensed under the Apache License, Version 2.0 (the "License");
-- you may not use this file except in compliance with the License.
-- You may obtain a copy of the License at
--
-- http://www.apache.org/licenses/LICENSE-2.0
--
-- Unless required by applicable law or agreed to in writing, software
-- distributed under the License is distributed on an "AS IS" BASIS,
-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-- See the License for the specific language governing permissions and
-- limitations under the License.
--

DELETE FROM jobAgentMetadata
WHERE jobAgentMetadata.id IN (
SELECT jobAgentMetadata.id
FROM jobAgentMetadata
INNER JOIN builds
ON
builds.id = jobAgentMetadata.jobId
WHERE
builds.state = 'Completed'
);

--//@UNDO

@@ -1,11 +1,11 @@
/*
* Copyright 2016 ThoughtWorks, Inc.
* Copyright 2017 ThoughtWorks, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
@@ -16,15 +16,17 @@

package com.thoughtworks.go.server.dao;

import com.thoughtworks.go.domain.EnvironmentVariable;
import com.thoughtworks.go.domain.EnvironmentVariableType;
import com.thoughtworks.go.domain.EnvironmentVariables;

import java.util.Collection;

public interface EnvironmentVariableDao {
void save(Long entityId, EnvironmentVariableType type, EnvironmentVariables variables);

EnvironmentVariables load(Long entityId, EnvironmentVariableType type);

enum EnvironmentVariableType {
Job,
Trigger
}
void deleteAll(Collection<EnvironmentVariable> variables);

}
@@ -1,11 +1,11 @@
/*
* Copyright 2016 ThoughtWorks, Inc.
* Copyright 2017 ThoughtWorks, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
@@ -17,6 +17,7 @@
package com.thoughtworks.go.server.dao;

import com.thoughtworks.go.domain.EnvironmentVariable;
import com.thoughtworks.go.domain.EnvironmentVariableType;
import com.thoughtworks.go.domain.EnvironmentVariables;
import com.thoughtworks.go.server.transaction.TransactionTemplate;
import org.hibernate.Criteria;
@@ -29,6 +30,7 @@
import org.springframework.transaction.support.TransactionCallback;
import org.springframework.transaction.support.TransactionCallbackWithoutResult;

import java.util.Collection;
import java.util.List;

@Component
@@ -68,4 +70,16 @@ public Object doInTransaction(TransactionStatus transactionStatus) {
});
return new EnvironmentVariables(result);
}

@Override
public void deleteAll(final Collection<EnvironmentVariable> variables) {
transactionTemplate.execute(new TransactionCallbackWithoutResult() {
@Override
protected void doInTransactionWithoutResult(TransactionStatus status) {
for (EnvironmentVariable variable : variables) {
sessionFactory.getCurrentSession().delete(variable);
}
}
});
}
}

0 comments on commit 23eb465

Please sign in to comment.
You can’t perform that action at this time.