Permalink
Browse files

Merge pull request #658 from sonatype/nexus-5372-timeline-corruption

NEXUS-5372: Alternative fix
  • Loading branch information...
cstamas committed Nov 14, 2012
2 parents 5c460cf + 2f259b0 commit 0506410815487806c8c1cefc3b618cd5e4455fb7
Showing with 1,928 additions and 135 deletions.
  1. +34 −59 .../nexus-timeline-plugin/src/main/java/org/sonatype/timeline/internal/DefaultTimelinePersistor.java
  2. +14 −1 ...s/plugins/nexus-timeline-plugin/src/test/java/org/sonatype/timeline/AbstractTimelineTestCase.java
  3. +59 −25 nexus/plugins/nexus-timeline-plugin/src/test/java/org/sonatype/timeline/TimelineTest.java
  4. +0 −10 ...timeline-plugin/src/test/resources/crashed-could-not-add/persist/timeline.2009-08-28.21-11-13.dat
  5. +0 −13 ...timeline-plugin/src/test/resources/crashed-could-not-add/persist/timeline.2009-08-28.21-11-53.dat
  6. +213 −0 ...-plugin/src/test/resources/crashed-could-not-add/persist/timeline.2012-11-13.14-04-34+0100-v3.dat
  7. +189 −0 ...-plugin/src/test/resources/crashed-could-not-add/persist/timeline.2012-11-13.14-05-06+0100-v3.dat
  8. +0 −10 ...meline-plugin/src/test/resources/crashed-could-not-purge/persist/timeline.2009-08-28.21-11-13.dat
  9. +0 −13 ...meline-plugin/src/test/resources/crashed-could-not-purge/persist/timeline.2009-08-28.21-11-53.dat
  10. +213 −0 ...lugin/src/test/resources/crashed-could-not-purge/persist/timeline.2012-11-13.14-04-34+0100-v3.dat
  11. +189 −0 ...lugin/src/test/resources/crashed-could-not-purge/persist/timeline.2012-11-13.14-05-06+0100-v3.dat
  12. +0 −4 ...imeline-plugin/src/test/resources/crashed-could-not-read/persist/timeline.2009-04-21.22-04-32.dat
  13. +213 −0 ...plugin/src/test/resources/crashed-could-not-read/persist/timeline.2012-11-13.14-04-34+0100-v3.dat
  14. +189 −0 ...plugin/src/test/resources/crashed-could-not-read/persist/timeline.2012-11-13.14-05-06+0100-v3.dat
  15. BIN ...ine-plugin/src/test/resources/crashed-could-not-retrieve/persist/timeline.2009-08-26.19-52-30.dat
  16. +213 −0 ...in/src/test/resources/crashed-could-not-retrieve/persist/timeline.2012-11-13.14-04-34+0100-v3.dat
  17. +189 −0 ...in/src/test/resources/crashed-could-not-retrieve/persist/timeline.2012-11-13.14-05-06+0100-v3.dat
  18. +213 −0 ...gin/src/test/resources/crashed-persist-corrupted/persist/timeline.2012-11-13.14-04-34+0100-v3.dat
@@ -12,6 +12,8 @@
*/
package org.sonatype.timeline.internal;
+import java.io.BufferedInputStream;
+import java.io.BufferedOutputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
@@ -46,26 +48,20 @@
*/
public class DefaultTimelinePersistor
{
+ // == V3
- @Deprecated
- private static final String V1_DATA_FILE_NAME_DATE_FORMAT = "yyyy-MM-dd.HH-mm-ss";
+ private static final String V3_DATA_FILE_NAME_PREFIX = "timeline.";
- @Deprecated
- private static final Pattern V1_DATA_FILE_NAME_PATTERN =
- Pattern.compile( "^timeline\\.(\\d{4}-\\d{2}-\\d{2}\\.\\d{2}-\\d{2}-\\d{2})\\.dat$" );
+ private static final String V3_DATA_FILE_NAME_SUFFIX = "-v3.dat";
- // ==
-
- private static final String V2_DATA_FILE_NAME_PREFIX = "timeline.";
-
- private static final String V2_DATA_FILE_NAME_SUFFIX = ".dat";
+ private static final String V3_DATA_FILE_NAME_DATE_FORMAT = "yyyy-MM-dd.HH-mm-ssZ";
- private static final String V2_DATA_FILE_NAME_DATE_FORMAT = "yyyy-MM-dd.HH-mm-ssZ";
-
- private static final Pattern V2_DATA_FILE_NAME_PATTERN = Pattern.compile( "^" + V2_DATA_FILE_NAME_PREFIX.replace(
- ".", "\\." ) + "(\\d{4}-\\d{2}-\\d{2}\\.\\d{2}-\\d{2}-\\d{2}[+-]\\d{4})" + V2_DATA_FILE_NAME_SUFFIX.replace(
+ private static final Pattern V3_DATA_FILE_NAME_PATTERN = Pattern.compile( "^" + V3_DATA_FILE_NAME_PREFIX.replace(
+ ".", "\\." ) + "(\\d{4}-\\d{2}-\\d{2}\\.\\d{2}-\\d{2}-\\d{2}[+-]\\d{4})" + V3_DATA_FILE_NAME_SUFFIX.replace(
".", "\\." ) + "$" );
+ // ==
+
private int rollingIntervalMillis;
private File persistDirectory;
@@ -106,14 +102,12 @@ protected synchronized void persist( final TimelineRecord... records )
OutputStream out = null;
try
{
+ out = new BufferedOutputStream( new FileOutputStream( getDataFile(), true ) );
for ( TimelineRecord record : records )
{
- out = new FileOutputStream( getDataFile(), true );
- byte[] bytes = toProto( record ).toByteArray();
- out.write( bytes.length );
- out.write( bytes );
- out.flush();
+ toProto( record ).writeDelimitedTo( out );
}
+ out.flush();
}
finally
{
@@ -163,8 +157,7 @@ protected void readAllSinceDays( final int days, final TimelineCallback callback
{
public boolean accept( File dir, String fname )
{
- return V2_DATA_FILE_NAME_PATTERN.matcher( fname ).matches()
- || V1_DATA_FILE_NAME_PATTERN.matcher( fname ).matches();
+ return V3_DATA_FILE_NAME_PATTERN.matcher( fname ).matches();
}
} );
@@ -240,8 +233,9 @@ else if ( filePtr >= result.size() )
}
else
{
+ final File file = result.get( filePtr );
// jump to next file
- currentIterator = readFile( result.get( filePtr ) );
+ currentIterator = readFile( file );
filePtr++;
continue;
}
@@ -261,13 +255,13 @@ else if ( filePtr >= result.size() )
InputStream in = null;
try
{
- in = new FileInputStream( file );
- while ( in.available() > 0 )
+ in = new BufferedInputStream( new FileInputStream( file ) );
+ // V3 uses delimited format
+ TimelineRecord rec = fromProto( TimeLineRecordProtos.TimeLineRecord.parseDelimitedFrom( in ) );
+ while ( rec != null )
{
- int length = in.read();
- byte[] bytes = new byte[length];
- in.read( bytes, 0, length );
- result.add( fromProto( TimeLineRecordProtos.TimeLineRecord.parseFrom( bytes ) ) );
+ result.add( rec );
+ rec = fromProto( TimeLineRecordProtos.TimeLineRecord.parseDelimitedFrom( in ) );
}
}
catch ( Exception e )
@@ -276,23 +270,18 @@ else if ( filePtr >= result.size() )
}
finally
{
- if ( in != null )
- {
- try
- {
- in.close();
- }
- catch ( IOException e )
- {
- }
- }
+ IOUtil.close( in );
}
return result.iterator();
}
protected TimelineRecord fromProto( TimeLineRecordProtos.TimeLineRecord rec )
{
+ if ( rec == null )
+ {
+ return null;
+ }
final Map<String, String> dataMap = new HashMap<String, String>();
for ( TimeLineRecordProtos.TimeLineRecord.Data data : rec.getDataList() )
{
@@ -305,37 +294,23 @@ protected TimelineRecord fromProto( TimeLineRecordProtos.TimeLineRecord rec )
protected String buildTimestampedFileName()
{
- final SimpleDateFormat dateFormat = new SimpleDateFormat( V2_DATA_FILE_NAME_DATE_FORMAT );
+ final SimpleDateFormat dateFormat = new SimpleDateFormat( V3_DATA_FILE_NAME_DATE_FORMAT );
final StringBuilder fileName = new StringBuilder();
- fileName.append( V2_DATA_FILE_NAME_PREFIX ).append(
+ fileName.append( V3_DATA_FILE_NAME_PREFIX ).append(
dateFormat.format( new Date( System.currentTimeMillis() ) ) ).append(
- V2_DATA_FILE_NAME_SUFFIX );
+ V3_DATA_FILE_NAME_SUFFIX );
return fileName.toString();
}
protected long getTimestampedFileNameTimestamp( final File file )
{
- final Matcher fnMatcher = V2_DATA_FILE_NAME_PATTERN.matcher( file.getName() );
- if ( fnMatcher.matches() )
- {
- final String datePattern = fnMatcher.group( 1 );
- try
- {
- return new SimpleDateFormat( V2_DATA_FILE_NAME_DATE_FORMAT ).parse( datePattern ).getTime();
- }
- catch ( ParseException e )
- {
- // silently go to next try
- }
- }
-
- final Matcher oldFnMatcher = V1_DATA_FILE_NAME_PATTERN.matcher( file.getName() );
- if ( oldFnMatcher.matches() )
+ final Matcher v3FnMatcher = V3_DATA_FILE_NAME_PATTERN.matcher( file.getName() );
+ if ( v3FnMatcher.matches() )
{
- final String datePattern = oldFnMatcher.group( 1 );
+ final String datePattern = v3FnMatcher.group( 1 );
try
{
- return new SimpleDateFormat( V1_DATA_FILE_NAME_DATE_FORMAT ).parse( datePattern ).getTime();
+ return new SimpleDateFormat( V3_DATA_FILE_NAME_DATE_FORMAT ).parse( datePattern ).getTime();
}
catch ( ParseException e )
{
@@ -25,6 +25,17 @@
public abstract class AbstractTimelineTestCase
extends InjectedTestCase
{
+
+ private static final String loremIpsum =
+ "Lorem ipsum dolor sit amet, consectetur adipiscing elit. Curabitur rutrum urna ac est sagittis in lacinia "
+ + "tortor porta. Maecenas accumsan hendrerit nulla vel lobortis. Phasellus purus sapien, fermentum non "
+ + "aliquet vitae, vulputate a nulla. Nunc a diam eget augue accumsan suscipit nec at tellus. Donec sit "
+ + "amet tellus mi, vitae gravida justo. Nulla iaculis ullamcorper sodales. Pellentesque ut viverra tellus. "
+ + "Pellentesque quis lacus velit. Nullam nec orci id ante pharetra dictum. In tincidunt fringilla metus, "
+ + "id faucibus ligula condimentum id. Aenean augue odio, auctor fermentum sodales non, gravida sit amet "
+ + "diam. Maecenas ac dolor at lorem ullamcorper pretium convallis ut felis. Mauris ut nisi ut leo "
+ + "fringilla auctor sed et lectus.";
+
protected DefaultTimeline timeline;
@Override
@@ -68,18 +79,20 @@ protected TimelineRecord createTimelineRecord( final long ts )
data.put( "k1", "v1" );
data.put( "k2", "v2" );
data.put( "k3", "v3" );
+ data.put( "k4", loremIpsum );
return createTimelineRecord( ts, "type", "subType", data );
}
protected TimelineRecord createTimelineRecord( final long ts, final String type, final String subType,
- final Map<String, String> data )
+ final Map<String, String> data )
{
return new TimelineRecord( ts, type, subType, data );
}
public static class AsList
implements TimelineCallback
{
+
private final ArrayList<TimelineRecord> records = new ArrayList<TimelineRecord>();
@Override
@@ -13,6 +13,7 @@
package org.sonatype.timeline;
import java.io.File;
+import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
@@ -26,6 +27,7 @@
public class TimelineTest
extends AbstractTimelineTestCase
{
+
protected File persistDirectory;
protected File indexDirectory;
@@ -109,21 +111,26 @@ public void testRepairIndexCouldNotRead()
FileUtils.copyDirectoryStructure( crashedPersistDir, persistDirectory );
FileUtils.copyDirectoryStructure( carshedIndexDir, indexDirectory );
- timeline.start( new TimelineConfiguration( persistDirectory, indexDirectory ) );
-
- Map<String, String> data = new HashMap<String, String>();
- data.put( "k1", "v1" );
- data.put( "k2", "v2" );
- data.put( "k3", "v3" );
+ // as time passes, the timestamps in test persist files will fall out of default 30 day, so we say restore all
+ timeline.start( new TimelineConfiguration( persistDirectory, indexDirectory,
+ TimelineConfiguration.DEFAULT_ROLLING_INTERVAL_MILLIS,
+ Integer.MAX_VALUE ) );
- Set<String> types = new HashSet<String>();
- types.add( "typeA" );
- AsList cb = new AsList();
- timeline.retrieve( 0, 10, types, null, null, cb );
- List<TimelineRecord> results = cb.getRecords();
-
- assertEquals( 1, results.size() );
- assertEquals( data, results.get( 0 ).getData() );
+ {
+ // all records have this type, so we should have them as many as we asked for but max 100
+ // as many records are in test persist directory
+ AsList cb = new AsList();
+ timeline.retrieve( 0, 1000, Collections.singleton( "type" ), null, null, cb );
+ List<TimelineRecord> results = cb.getRecords();
+ assertEquals( 100, results.size() );
+ }
+ {
+ // no records have this type, so we should have 0
+ AsList cb = new AsList();
+ timeline.retrieve( 0, 10, Collections.singleton( "badtype" ), null, null, cb );
+ List<TimelineRecord> results = cb.getRecords();
+ assertEquals( 0, results.size() );
+ }
}
@Test
@@ -135,11 +142,26 @@ public void testRepairIndexCouldNotRetrieve()
FileUtils.copyDirectoryStructure( crashedPersistDir, persistDirectory );
FileUtils.copyDirectoryStructure( carshedIndexDir, indexDirectory );
- timeline.start( new TimelineConfiguration( persistDirectory, indexDirectory ) );
+ // as time passes, the timestamps in test persist files will fall out of default 30 day, so we say restore all
+ timeline.start( new TimelineConfiguration( persistDirectory, indexDirectory,
+ TimelineConfiguration.DEFAULT_ROLLING_INTERVAL_MILLIS,
+ Integer.MAX_VALUE ) );
- AsList cb = new AsList();
- timeline.retrieve( 0, 10, null, null, null, cb );
- assertTrue( cb.getRecords().size() > 0 );
+ {
+ // all records have this type, so we should have them as many as we asked for but max 100
+ // as many records are in test persist directory
+ AsList cb = new AsList();
+ timeline.retrieve( 0, 1000, Collections.singleton( "type" ), null, null, cb );
+ List<TimelineRecord> results = cb.getRecords();
+ assertEquals( 100, results.size() );
+ }
+ {
+ // no records have this type, so we should have 0
+ AsList cb = new AsList();
+ timeline.retrieve( 0, 10, Collections.singleton( "badtype" ), null, null, cb );
+ List<TimelineRecord> results = cb.getRecords();
+ assertEquals( 0, results.size() );
+ }
}
@Test
@@ -152,7 +174,10 @@ public void testRepairIndexCouldNotAdd()
FileUtils.copyDirectoryStructure( persistDir, persistDirectory );
FileUtils.copyDirectoryStructure( goodIndexDir, indexDirectory );
- timeline.start( new TimelineConfiguration( persistDirectory, indexDirectory ) );
+ // as time passes, the timestamps in test persist files will fall out of default 30 day, so we say restore all
+ timeline.start( new TimelineConfiguration( persistDirectory, indexDirectory,
+ TimelineConfiguration.DEFAULT_ROLLING_INTERVAL_MILLIS,
+ Integer.MAX_VALUE ) );
{
// add, this should pass without any exception
@@ -167,7 +192,10 @@ public void testRepairIndexCouldNotAdd()
timeline.stop();
cleanDirectory( indexDirectory );
FileUtils.copyDirectoryStructure( crashedIndexDir, indexDirectory );
- timeline.start( new TimelineConfiguration( persistDirectory, indexDirectory ) );
+ // as time passes, the timestamps in test persist files will fall out of default 30 day, so we say restore all
+ timeline.start( new TimelineConfiguration( persistDirectory, indexDirectory,
+ TimelineConfiguration.DEFAULT_ROLLING_INTERVAL_MILLIS,
+ Integer.MAX_VALUE ) );
{
// add again, this should also pass without any exception
@@ -189,16 +217,22 @@ public void testRepairIndexCouldNotPurge()
FileUtils.copyDirectoryStructure( persistDir, persistDirectory );
FileUtils.copyDirectoryStructure( goodIndexDir, indexDirectory );
- timeline.start( new TimelineConfiguration( persistDirectory, indexDirectory ) );
+ // as time passes, the timestamps in test persist files will fall out of default 30 day, so we say restore all
+ timeline.start( new TimelineConfiguration( persistDirectory, indexDirectory,
+ TimelineConfiguration.DEFAULT_ROLLING_INTERVAL_MILLIS,
+ Integer.MAX_VALUE ) );
- assertTrue( timeline.purge( System.currentTimeMillis(), null, null, null ) > 0 );
+ // here, the "good" index really contains 6 records only
+ assertEquals( 6, timeline.purge( System.currentTimeMillis(), null, null, null ) );
// pretend that when timeline is running, the index is manually changed
timeline.stop();
cleanDirectory( indexDirectory );
FileUtils.copyDirectoryStructure( crashedIndexDir, indexDirectory );
- timeline.start( new TimelineConfiguration( persistDirectory, indexDirectory ) );
-
- assertTrue( timeline.purge( System.currentTimeMillis(), null, null, null ) > 0 );
+ // as time passes, the timestamps in test persist files will fall out of default 30 day, so we say restore all
+ timeline.start( new TimelineConfiguration( persistDirectory, indexDirectory,
+ TimelineConfiguration.DEFAULT_ROLLING_INTERVAL_MILLIS,
+ Integer.MAX_VALUE ) );
+ assertEquals( 100, timeline.purge( System.currentTimeMillis(), null, null, null ) );
}
}
@@ -1,10 +0,0 @@
-���$SYSTEMCONFIG"/
-message$Nexus configuration changed/updated."
-actionCONFIG"$
-date2009-08-28 21:11:13.389+0800�˫���$SYSTEMCONFIG"/
-message$Nexus configuration changed/updated."
-actionCONFIG"$
-date2009-08-28 21:11:14.123+0800����$SYSTEMBOOT"6
-message+Starting Nexus (version 1.4.0-SNAPSHOT OSS)"
-actionBOOT"$
-date2009-08-28 21:11:14.126+0800
@@ -1,13 +0,0 @@
-��ވ��$SYSTEMCONFIG"/
-message$Nexus configuration changed/updated."
-actionCONFIG"$
-date2009-08-28 21:11:53.400+0800��䈉�$SYSTEMCONFIG"/
-message$Nexus configuration changed/updated."
-actionCONFIG"$
-date2009-08-28 21:11:54.165+0800��䈉�$SYSTEMBOOT"6
-message+Starting Nexus (version 1.4.0-SNAPSHOT OSS)"
-actionBOOT"$
-date2009-08-28 21:11:54.169+0800���$SYSTEMBOOT"6
-message+Stopping Nexus (version 1.4.0-SNAPSHOT OSS)"
-actionBOOT"$
-date2009-08-28 21:14:24.338+0800
Oops, something went wrong.

0 comments on commit 0506410

Please sign in to comment.