From 6011e4e9bce16f63df066a9ca0073c3b31fbdd0b Mon Sep 17 00:00:00 2001 From: John Casey Date: Thu, 24 Oct 2019 13:04:04 -0500 Subject: [PATCH 1/3] only do deploy when we're doing an image build, and add timestamps to Folo tracking records --- Jenkinsfile | 32 +++++++++++-------- .../indy/folo/ctl/FoloAdminController.java | 1 + .../indy/folo/data/FoloRecordCache.java | 16 ++++++++-- .../indy/folo/dto/TrackedContentEntryDTO.java | 13 ++++++++ .../indy/folo/model/TrackedContentEntry.java | 28 +++++++++++++++- 5 files changed, 74 insertions(+), 16 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 579c7607b7..87c85a45ba 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -22,19 +22,6 @@ pipeline { sh 'mvn -B -V verify -Prun-its -Pci' } } - stage('Deploy') { - when { branch 'master' } - steps { - echo "Deploy" - sh 'mvn help:effective-settings -B -V deploy -e' - } - } - stage('Archive') { - steps { - echo "Archive" - archiveArtifacts artifacts: "$artifact", fingerprint: true - } - } stage('Check Image Build Hook') { when { expression { env.IMG_BUILD_HOOKS != null } @@ -54,6 +41,25 @@ pipeline { } } } + stage('Deploy') { + when { + allOf { + expression { img_build_hook != null } + expression { env.CHANGE_ID == null } // Not pull request + branch 'master' + } + } + steps { + echo "Deploy" + sh 'mvn help:effective-settings -B -V deploy -e' + } + } + stage('Archive') { + steps { + echo "Archive" + archiveArtifacts artifacts: "$artifact", fingerprint: true + } + } stage('Build & Push Image') { when { allOf { diff --git a/addons/folo/common/src/main/java/org/commonjava/indy/folo/ctl/FoloAdminController.java b/addons/folo/common/src/main/java/org/commonjava/indy/folo/ctl/FoloAdminController.java index f0cbb2fcec..0ee8617333 100644 --- a/addons/folo/common/src/main/java/org/commonjava/indy/folo/ctl/FoloAdminController.java +++ b/addons/folo/common/src/main/java/org/commonjava/indy/folo/ctl/FoloAdminController.java @@ -334,6 +334,7 @@ private TrackedContentEntryDTO constructContentEntryDTO( final TrackedContentEnt entryDTO.setSha1( entry.getSha1() ); entryDTO.setSha256( entry.getSha256() ); entryDTO.setSize( entry.getSize() ); + entryDTO.setTimestamps( entry.getTimestamps() ); return entryDTO; } diff --git a/addons/folo/common/src/main/java/org/commonjava/indy/folo/data/FoloRecordCache.java b/addons/folo/common/src/main/java/org/commonjava/indy/folo/data/FoloRecordCache.java index 1f45776491..073ba5d707 100644 --- a/addons/folo/common/src/main/java/org/commonjava/indy/folo/data/FoloRecordCache.java +++ b/addons/folo/common/src/main/java/org/commonjava/indy/folo/data/FoloRecordCache.java @@ -107,8 +107,20 @@ public synchronized boolean recordArtifact( final TrackedContentEntry entry ) Logger logger = LoggerFactory.getLogger( getClass() ); logger.debug( "Adding tracking entry: {}", entry ); - inProgressRecordCache.put( entry, entry ); - return true; + return inProgressRecordCache.executeCache( (cache)->{ + TrackedContentEntry existing = cache.get( entry ); + if ( existing != null ) + { + existing.merge( entry ); + cache.put( existing, existing ); + } + else + { + cache.put( entry, entry ); + } + + return true; + } ); } @Measure( timers = @MetricNamed( DEFAULT ) ) diff --git a/addons/folo/model-java/src/main/java/org/commonjava/indy/folo/dto/TrackedContentEntryDTO.java b/addons/folo/model-java/src/main/java/org/commonjava/indy/folo/dto/TrackedContentEntryDTO.java index 810d3565d3..7391244cc4 100644 --- a/addons/folo/model-java/src/main/java/org/commonjava/indy/folo/dto/TrackedContentEntryDTO.java +++ b/addons/folo/model-java/src/main/java/org/commonjava/indy/folo/dto/TrackedContentEntryDTO.java @@ -20,6 +20,8 @@ import org.commonjava.indy.model.core.AccessChannel; import org.commonjava.indy.model.core.StoreKey; +import java.util.Set; + public class TrackedContentEntryDTO implements Comparable { @@ -48,6 +50,8 @@ public class TrackedContentEntryDTO private Long size; + private Set timestamps; + public TrackedContentEntryDTO() { } @@ -238,4 +242,13 @@ public String toString() storeKey, accessChannel, path, originUrl, localUrl, size, md5, sha256 ); } + public Set getTimestamps() + { + return timestamps; + } + + public void setTimestamps( final Set timestamps ) + { + this.timestamps = timestamps; + } } diff --git a/addons/folo/model-java/src/main/java/org/commonjava/indy/folo/model/TrackedContentEntry.java b/addons/folo/model-java/src/main/java/org/commonjava/indy/folo/model/TrackedContentEntry.java index 20650fdb57..316c9dfe30 100644 --- a/addons/folo/model-java/src/main/java/org/commonjava/indy/folo/model/TrackedContentEntry.java +++ b/addons/folo/model-java/src/main/java/org/commonjava/indy/folo/model/TrackedContentEntry.java @@ -23,6 +23,9 @@ import java.io.IOException; import java.io.ObjectInput; import java.io.ObjectOutput; +import java.util.Collections; +import java.util.HashSet; +import java.util.Set; import static org.commonjava.indy.model.core.AccessChannel.GENERIC_PROXY; import static org.commonjava.indy.pkg.PackageTypeConstants.PKG_TYPE_GENERIC_HTTP; @@ -31,7 +34,7 @@ public class TrackedContentEntry implements Comparable,Externalizable { - private static final int VERSION = 2; + private static final int VERSION = 3; private TrackingKey trackingKey; @@ -55,6 +58,8 @@ public class TrackedContentEntry private long index = System.currentTimeMillis(); + private Set timestamps; + public TrackedContentEntry() { } @@ -74,6 +79,7 @@ public TrackedContentEntry( final TrackingKey trackingKey, final StoreKey storeK this.sha1=sha1; this.sha256=sha256; this.size = size; + this.timestamps = new HashSet<>( Collections.singleton( System.currentTimeMillis() ) ); } public String getOriginUrl() @@ -278,6 +284,7 @@ public void writeExternal( final ObjectOutput out ) out.writeObject( sha256 == null ? "" : sha256 ); out.writeObject( size ); out.writeLong( index ); + out.writeObject( timestamps ); } @Override @@ -357,6 +364,25 @@ else if ( whatIsThis instanceof TrackingKey ) index = in.readLong(); + if ( version > 2 ) + { + final Set tstamps = (Set) in.readObject(); + timestamps = tstamps; + } + } + + public Set getTimestamps() + { + return timestamps; } + public void setTimestamps( final Set timestamps ) + { + this.timestamps = timestamps; + } + + public void merge( TrackedContentEntry from ) + { + this.timestamps.addAll( from.timestamps ); + } } From d799d49ae7ff0c4bb07da02f0f2b2e7dd2cd4662 Mon Sep 17 00:00:00 2001 From: John Casey Date: Thu, 24 Oct 2019 17:18:42 -0500 Subject: [PATCH 2/3] Adding unit tests for version change in TrackedContentEntry --- .../folo/model/TrackedContentEntryTest.java | 56 +++ .../folo/model/TrackedContentEntryV2.java | 363 ++++++++++++++++++ 2 files changed, 419 insertions(+) create mode 100644 addons/folo/model-java/src/test/java/org/commonjava/indy/folo/model/TrackedContentEntryV2.java diff --git a/addons/folo/model-java/src/test/java/org/commonjava/indy/folo/model/TrackedContentEntryTest.java b/addons/folo/model-java/src/test/java/org/commonjava/indy/folo/model/TrackedContentEntryTest.java index f7e08d9aae..a707454101 100644 --- a/addons/folo/model-java/src/test/java/org/commonjava/indy/folo/model/TrackedContentEntryTest.java +++ b/addons/folo/model-java/src/test/java/org/commonjava/indy/folo/model/TrackedContentEntryTest.java @@ -24,6 +24,7 @@ import java.io.IOException; import java.io.ObjectInputStream; import java.io.ObjectOutputStream; +import java.util.Set; import static org.commonjava.indy.folo.model.StoreEffect.DOWNLOAD; import static org.commonjava.indy.model.core.AccessChannel.GENERIC_PROXY; @@ -33,6 +34,7 @@ import static org.commonjava.indy.pkg.PackageTypeConstants.PKG_TYPE_GENERIC_HTTP; import static org.commonjava.indy.pkg.PackageTypeConstants.PKG_TYPE_MAVEN; import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.CoreMatchers.nullValue; import static org.junit.Assert.assertThat; public class TrackedContentEntryTest @@ -86,6 +88,59 @@ public void externalizeAsV1_readCurrentVersion() assertThat( out.getSha256(), equalTo( test.getSha256() ) ); } + /** + * We have to hack this test a bit in order to test the ability to deserialize this first version of + * TrackedContentEntry. + * @throws IOException + * @throws ClassNotFoundException + */ + @Test + public void externalizeAsV2_readCurrentVersion() + throws IOException, ClassNotFoundException + { + TrackedContentEntryV2 ev2 = new TrackedContentEntryV2( new TrackingKey( "test-key" ), + new StoreKey( PKG_TYPE_GENERIC_HTTP, remote, + "some-upstream" ), GENERIC_PROXY, + "http://some.upstream.url/path/to/file", "path/to/file", + DOWNLOAD, 10101010L, "aaaafffffccccceeeeddd", + "bbbcccceeeedddaaaaa", "aaadadaaaadaeee" ); + + TrackedContentEntry out = new TrackedContentEntry( new TrackingKey( "test-key2" ), + new StoreKey( PKG_TYPE_MAVEN, hosted, + "some-upstream2" ), MAVEN_REPO, + "http://some.upstream.url/path/to/file2", "path/to/file2", + DOWNLOAD, 10101011L, "aaaafffffccccceeeedddfffffff", + "bbbcccceeeedddaaaaaffffffff", "aaadadaaaadaeeeffffffff" ); + + // nullify this to make sure that reading from the old version of the object doesn't set it. + out.setTimestamps( null ); + + TrackedContentEntry test = + new TrackedContentEntry( ev2.getTrackingKey(), ev2.getStoreKey(), ev2.getAccessChannel(), + ev2.getOriginUrl(), ev2.getPath(), ev2.getEffect(), ev2.getSize(), + ev2.getMd5(), ev2.getSha1(), ev2.getSha256() ); + + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + ObjectOutputStream oos = new ObjectOutputStream( baos ); + ev2.writeExternal( oos ); + oos.flush(); + + ObjectInputStream oin = new ObjectInputStream( new ByteArrayInputStream( baos.toByteArray() ) ); + out.readExternal( oin ); + + assertThat( out.getTrackingKey(), equalTo( test.getTrackingKey() ) ); + assertThat( out.getStoreKey(), equalTo( test.getStoreKey() ) ); + assertThat( out.getAccessChannel(), equalTo( test.getAccessChannel() ) ); + assertThat( out.getOriginUrl(), equalTo( test.getOriginUrl() ) ); + assertThat( out.getPath(), equalTo( test.getPath() ) ); + assertThat( out.getEffect(), equalTo( test.getEffect() ) ); + assertThat( out.getSize(), equalTo( test.getSize() ) ); + assertThat( out.getMd5(), equalTo( test.getMd5() ) ); + assertThat( out.getSha1(), equalTo( test.getSha1() ) ); + assertThat( out.getSha256(), equalTo( test.getSha256() ) ); + assertThat( out.getTimestamps(), nullValue() ); + } + @Test public void serializeRoundTrip_CurrentVersion() throws IOException, ClassNotFoundException @@ -120,5 +175,6 @@ public void serializeRoundTrip_CurrentVersion() assertThat( out.getMd5(), equalTo( test.getMd5() ) ); assertThat( out.getSha1(), equalTo( test.getSha1() ) ); assertThat( out.getSha256(), equalTo( test.getSha256() ) ); + assertThat( out.getTimestamps(), equalTo( test.getTimestamps() ) ); } } diff --git a/addons/folo/model-java/src/test/java/org/commonjava/indy/folo/model/TrackedContentEntryV2.java b/addons/folo/model-java/src/test/java/org/commonjava/indy/folo/model/TrackedContentEntryV2.java new file mode 100644 index 0000000000..e1a6e83727 --- /dev/null +++ b/addons/folo/model-java/src/test/java/org/commonjava/indy/folo/model/TrackedContentEntryV2.java @@ -0,0 +1,363 @@ +/** + * Copyright (C) 2011-2019 Red Hat, Inc. (https://github.com/Commonjava/indy) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.commonjava.indy.folo.model; + +import org.commonjava.indy.model.core.AccessChannel; +import org.commonjava.indy.model.core.StoreKey; +import org.commonjava.indy.model.core.StoreType; + +import java.io.Externalizable; +import java.io.IOException; +import java.io.ObjectInput; +import java.io.ObjectOutput; +import java.util.Collections; +import java.util.HashSet; +import java.util.Set; + +import static org.commonjava.indy.model.core.AccessChannel.GENERIC_PROXY; +import static org.commonjava.indy.pkg.PackageTypeConstants.PKG_TYPE_GENERIC_HTTP; +import static org.commonjava.indy.pkg.PackageTypeConstants.PKG_TYPE_MAVEN; + +public class TrackedContentEntryV2 + implements Comparable, Externalizable +{ + private static final int VERSION = 2; + + private TrackingKey trackingKey; + + private StoreKey storeKey; + + private AccessChannel accessChannel; + + private String path; + + private String originUrl; + + private StoreEffect effect; + + private String md5; + + private String sha256; + + private String sha1; + + private Long size; + + private long index = System.currentTimeMillis(); + + public TrackedContentEntryV2() + { + } + + public TrackedContentEntryV2( final TrackingKey trackingKey, final StoreKey storeKey, + final AccessChannel accessChannel, final String originUrl, final String path, + final StoreEffect effect, final Long size, + final String md5, final String sha1, final String sha256 ) + { + this.trackingKey = trackingKey; + this.storeKey = storeKey; + this.accessChannel = accessChannel; + this.path = path; + this.originUrl = originUrl; + this.effect = effect; + this.md5=md5; + this.sha1=sha1; + this.sha256=sha256; + this.size = size; + } + + public String getOriginUrl() + { + return originUrl; + } + + public String getMd5() + { + return md5; + } + + public String getSha256() + { + return sha256; + } + + public String getSha1() + { + return sha1; + } + + public StoreKey getStoreKey() + { + return storeKey; + } + + public AccessChannel getAccessChannel() + { + return accessChannel; + } + + public String getPath() + { + return path; + } + + public TrackingKey getTrackingKey() + { + return trackingKey; + } + + public StoreEffect getEffect() + { + return effect; + } + + public Long getSize() + { + return size; + } + + public long getIndex() + { + return index; + } + + public void setStoreKey( StoreKey storeKey ) + { + this.storeKey = storeKey; + } + + public void setOriginUrl( String originUrl ) + { + this.originUrl = originUrl; + } + + @Override + public int compareTo( final TrackedContentEntryV2 other ) + { + int comp = trackingKey.getId().compareTo( other.getTrackingKey().getId() ); + if ( comp == 0 ) + { + comp = storeKey.compareTo( other.getStoreKey() ); + } + if ( comp == 0 ) + { + comp = accessChannel.compareTo( other.getAccessChannel() ); + } + if ( comp == 0 ) + { + comp = path.compareTo( other.getPath() ); + } + if ( comp == 0 ) + { + comp = effect.compareTo( other.getEffect() ); + } + + return comp; + } + + @Override + public int hashCode() + { + final int prime = 31; + int result = 1; + result = prime * result + ( ( trackingKey == null ) ? 0 : trackingKey.hashCode() ); + result = prime * result + ( ( path == null ) ? 0 : path.hashCode() ); + result = prime * result + ( ( storeKey == null ) ? 0 : storeKey.hashCode() ); + result = prime * result + ( ( accessChannel == null ) ? 0 : accessChannel.hashCode() ); + result = prime * result + ( ( effect == null ) ? 0 : effect.hashCode() ); + return result; + } + + @Override + public boolean equals( final Object obj ) + { + if ( this == obj ) + { + return true; + } + if ( obj == null ) + { + return false; + } + if ( getClass() != obj.getClass() ) + { + return false; + } + final TrackedContentEntryV2 other = (TrackedContentEntryV2) obj; + if ( trackingKey == null ) + { + if ( other.trackingKey != null ) + { + return false; + } + } + else if ( !trackingKey.equals( other.trackingKey ) ) + { + return false; + } + if ( path == null ) + { + if ( other.path != null ) + { + return false; + } + } + else if ( !path.equals( other.path ) ) + { + return false; + } + if ( storeKey == null ) + { + if ( other.storeKey != null ) + { + return false; + } + } + else if ( !storeKey.equals( other.storeKey ) ) + { + return false; + } + if ( accessChannel == null ) + { + if ( other.accessChannel != null ) + { + return false; + } + } + else if ( !accessChannel.equals( other.accessChannel ) ) + { + return false; + } + if ( effect == null ) + { + if ( other.effect != null ) + { + return false; + } + } + else if ( !effect.equals( other.effect ) ) + { + return false; + } + return true; + } + + @Override + public String toString() + { + return String.format( + "TrackedContentEntry [\n trackingKey=%s\n storeKey=%s\n accessChannel=%s\n path=%s\n originUrl=%s\n effect=%s\n md5=%s\n sha1=%s\n sha256=%s\nObject hashcode=%s\n]", + trackingKey, storeKey, accessChannel, path, originUrl, effect, md5, sha1, sha256, super.hashCode() ); + } + + @Override + public void writeExternal( final ObjectOutput out ) + throws IOException + { + out.writeObject( Integer.toString( VERSION ) ); + out.writeObject( trackingKey ); + out.writeObject( storeKey.getPackageType() ); + out.writeObject( storeKey.getName() ); + out.writeObject( storeKey.getType().name() ); + out.writeObject( accessChannel.name() ); + out.writeObject( path == null ? "" : path ); + out.writeObject( originUrl == null ? "" : originUrl ); + out.writeObject( effect == null ? "" : effect.name() ); + out.writeObject( md5 == null ? "" : md5 ); + out.writeObject( sha1 == null ? "" : sha1 ); + out.writeObject( sha256 == null ? "" : sha256 ); + out.writeObject( size ); + out.writeLong( index ); + } + + @Override + public void readExternal( final ObjectInput in ) + throws IOException, ClassNotFoundException + { + // This is a little awkward. The original version didn't have a version constant, so it wasn't possible + // to just read it from the data stream and use it to guide the deserialization process. Instead, + // we have to read the first object and determine whether it's the object version or the tracking key, + // which was the first field in the serialized data stream back in the first version of this class. + Object whatIsThis = in.readObject(); + + int version; + String packageType; + + if ( whatIsThis == null ) + { + // we see NumberFormatException: For input string: "null" in parseInt. It might be because we forget + // to persist the trackingKey for some reason. In this case, we just ignore it and continues + version = 1; + trackingKey = null; + packageType = PKG_TYPE_MAVEN; + } + else if ( whatIsThis instanceof TrackingKey ) + { + version = 1; + trackingKey = (TrackingKey) whatIsThis; + packageType = PKG_TYPE_MAVEN; + } + else + { + version = Integer.parseInt( String.valueOf( whatIsThis ) ); + trackingKey = (TrackingKey) in.readObject(); + packageType = (String) in.readObject(); + } + + // TODO: We should make future versioning / deserialization decisions based on the version we read / infer above + if ( version > VERSION ) + { + throw new IOException( + "This class is of an older version: " + VERSION + " vs. the version read from the data stream: " + + version + ". Cannot deserialize." ); + } + + final String storeKeyName = (String) in.readObject(); + final StoreType storeType = StoreType.get( (String) in.readObject() ); + + final String accessChannelStr = (String) in.readObject(); + accessChannel = "".equals( accessChannelStr ) ? null : AccessChannel.valueOf( accessChannelStr ); + + if ( version == 1 && accessChannel == GENERIC_PROXY ) + { + packageType = PKG_TYPE_GENERIC_HTTP; + } + + storeKey = new StoreKey( packageType, storeType, storeKeyName ); + + final String pathStr = (String) in.readObject(); + path = "".equals( pathStr ) ? null : pathStr; + + final String originUrlStr = (String) in.readObject(); + originUrl = "".equals( originUrlStr ) ? null : originUrlStr; + + final String effectStr = (String) in.readObject(); + effect = "".equals( effectStr ) ? null : StoreEffect.valueOf( effectStr ); + + final String md5Str = (String) in.readObject(); + md5 = "".equals( md5Str ) ? null : md5Str; + + final String sha1Str = (String) in.readObject(); + sha1 = "".equals( sha1Str ) ? null : sha1Str; + + final String sha256Str = (String) in.readObject(); + sha256 = "".equals( sha256Str ) ? null : sha256Str; + + size = (Long) in.readObject(); + + index = in.readLong(); + } +} From 8345d1a8aaeb00800b702df45dd0049ac0c3303c Mon Sep 17 00:00:00 2001 From: John Casey Date: Tue, 29 Oct 2019 14:27:02 -0500 Subject: [PATCH 3/3] Fixing merge logic in TrackedContentEntry and fixing serialVersionUID so we can use Externalizable versioning for deser --- .../content/DuplicateStoreAndVerifyTrackedRecordTest.java | 2 +- .../org/commonjava/indy/folo/model/TrackedContentEntry.java | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/addons/folo/ftests/src/main/java/org/commonjava/indy/folo/ftest/content/DuplicateStoreAndVerifyTrackedRecordTest.java b/addons/folo/ftests/src/main/java/org/commonjava/indy/folo/ftest/content/DuplicateStoreAndVerifyTrackedRecordTest.java index 0034e5a614..8a3be8080e 100644 --- a/addons/folo/ftests/src/main/java/org/commonjava/indy/folo/ftest/content/DuplicateStoreAndVerifyTrackedRecordTest.java +++ b/addons/folo/ftests/src/main/java/org/commonjava/indy/folo/ftest/content/DuplicateStoreAndVerifyTrackedRecordTest.java @@ -66,7 +66,7 @@ public void run() throws Exception Set uploads = report.getUploads(); uploads.forEach( et -> { System.out.println( ">>> md5: " + et.getMd5() + ", size=" + et.getSize() ); - assertThat( et.getSize(), equalTo( (long) b2.length ) ); + assertThat( "Mismatched size for: " + et.getPath(), et.getSize(), equalTo( (long) b2.length ) ); } ); } diff --git a/addons/folo/model-java/src/main/java/org/commonjava/indy/folo/model/TrackedContentEntry.java b/addons/folo/model-java/src/main/java/org/commonjava/indy/folo/model/TrackedContentEntry.java index 316c9dfe30..4a5e064088 100644 --- a/addons/folo/model-java/src/main/java/org/commonjava/indy/folo/model/TrackedContentEntry.java +++ b/addons/folo/model-java/src/main/java/org/commonjava/indy/folo/model/TrackedContentEntry.java @@ -34,6 +34,8 @@ public class TrackedContentEntry implements Comparable,Externalizable { + private static final long serialVersionUID = 6469004486206600578L; + private static final int VERSION = 3; private TrackingKey trackingKey; @@ -383,6 +385,10 @@ public void setTimestamps( final Set timestamps ) public void merge( TrackedContentEntry from ) { + this.md5 = from.md5; + this.sha1 = from.sha1; + this.sha256 = from.sha256; + this.size = from.size; this.timestamps.addAll( from.timestamps ); } }