Skip to content
This repository

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse code

Merge pull request #792 from apcj/pwp-no-wait

PersistenceWindowPool.refreshBricks() blocking
  • Loading branch information...
commit 21e95403a8f2668dd2f969249b4954ec81f6d4c3 2 parents 0beff1c + d1075b0
Mattias Persson tinwelint authored
112 kernel/src/main/java/org/neo4j/kernel/impl/nioneo/store/PersistenceWindowPool.java
@@ -28,6 +28,9 @@
28 28 import java.util.List;
29 29 import java.util.concurrent.ConcurrentHashMap;
30 30 import java.util.concurrent.ConcurrentMap;
  31 +import java.util.concurrent.atomic.AtomicBoolean;
  32 +import java.util.concurrent.atomic.AtomicInteger;
  33 +import java.util.concurrent.atomic.AtomicLong;
31 34 import java.util.logging.Level;
32 35 import java.util.logging.Logger;
33 36
@@ -70,6 +73,11 @@
70 73
71 74 private final boolean readOnly;
72 75
  76 + private final AtomicBoolean refreshing = new AtomicBoolean();
  77 + private final AtomicInteger avertedRefreshes = new AtomicInteger();
  78 + private final AtomicLong refreshTime = new AtomicLong();
  79 + private final AtomicInteger refreshes = new AtomicInteger();
  80 +
73 81 /**
74 82 * Create new pool for a store.
75 83 *
@@ -97,6 +105,7 @@ public PersistenceWindowPool( String storeName, int blockSize,
97 105 this.mapMode = readOnly ? MapMode.READ_ONLY : MapMode.READ_WRITE;
98 106 setupBricks();
99 107 dumpStatus();
  108 + System.out.println( getStats().toString() );
100 109 }
101 110
102 111 /**
@@ -465,50 +474,72 @@ private void refreshBricks()
465 474 if ( brickMiss < REFRESH_BRICK_COUNT || brickSize <= 0 )
466 475 return;
467 476
468   - synchronized ( this )
  477 + if ( refreshing.compareAndSet( false, true ) )
469 478 {
470   - brickMiss = 0;
471   - Pair<List<BrickElement>, List<BrickElement>> currentMappings = gatherMappedVersusUnmappedWindows();
472   - List<BrickElement> mappedBricks = currentMappings.first();
473   - List<BrickElement> unmappedBricks = currentMappings.other();
474   -
475   - // Fill up unused memory, i.e. map unmapped bricks as much as available memory allows
476   - // and request patterns signals. Start the loop from the end of the array where the
477   - // bricks with highest hit ratio are.
478   - int unmappedIndex = unmappedBricks.size() - 1;
479   - while ( memUsed + brickSize <= availableMem && unmappedIndex >= 0 )
  479 + // No one is doing refresh right now, go ahead and do it
  480 + try
480 481 {
481   - BrickElement unmappedBrick = unmappedBricks.get( unmappedIndex-- );
482   - if ( unmappedBrick.getHit() == 0 )
483   - // We have more memory available, but no more windows have actually
484   - // been requested so don't map unused random windows.
485   - return;
486   -
487   - allocateNewWindow( unmappedBrick );
  482 + long t = System.currentTimeMillis();
  483 + doRefreshBricks();
  484 + refreshes.incrementAndGet();
  485 + refreshTime.addAndGet( System.currentTimeMillis()-t );
488 486 }
  487 + finally
  488 + {
  489 + refreshing.set( false );
  490 + }
  491 + }
  492 + else
  493 + {
  494 + // Another thread is doing refresh right now, trust it to refresh the bricks
  495 + // and just go about my business.
  496 + avertedRefreshes.incrementAndGet();
  497 + }
  498 + }
  499 +
  500 + private synchronized void doRefreshBricks()
  501 + {
  502 + brickMiss = 0;
  503 + Pair<List<BrickElement>, List<BrickElement>> currentMappings = gatherMappedVersusUnmappedWindows();
  504 + List<BrickElement> mappedBricks = currentMappings.first();
  505 + List<BrickElement> unmappedBricks = currentMappings.other();
  506 +
  507 + // Fill up unused memory, i.e. map unmapped bricks as much as available memory allows
  508 + // and request patterns signals. Start the loop from the end of the array where the
  509 + // bricks with highest hit ratio are.
  510 + int unmappedIndex = unmappedBricks.size() - 1;
  511 + while ( memUsed + brickSize <= availableMem && unmappedIndex >= 0 )
  512 + {
  513 + BrickElement unmappedBrick = unmappedBricks.get( unmappedIndex-- );
  514 + if ( unmappedBrick.getHit() == 0 )
  515 + // We have more memory available, but no more windows have actually
  516 + // been requested so don't map unused random windows.
  517 + return;
489 518
490   - // Switch bad/unused mappings. Start iterating over mapped bricks
491   - // from the beginning (those with lowest hit ratio) and unmapped from the end
492   - // (or rather where the fill-up-unused-memory loop above left off) where we've
493   - // got the unmapped bricks with highest hit ratio.
494   - int mappedIndex = 0;
495   - while ( unmappedIndex >= 0 && mappedIndex < mappedBricks.size() )
  519 + allocateNewWindow( unmappedBrick );
  520 + }
  521 +
  522 + // Switch bad/unused mappings. Start iterating over mapped bricks
  523 + // from the beginning (those with lowest hit ratio) and unmapped from the end
  524 + // (or rather where the fill-up-unused-memory loop above left off) where we've
  525 + // got the unmapped bricks with highest hit ratio.
  526 + int mappedIndex = 0;
  527 + while ( unmappedIndex >= 0 && mappedIndex < mappedBricks.size() )
  528 + {
  529 + BrickElement mappedBrick = mappedBricks.get( mappedIndex++ );
  530 + BrickElement unmappedBrick = unmappedBricks.get( unmappedIndex-- );
  531 + if ( mappedBrick.getHit() >= unmappedBrick.getHit() )
  532 + // We've passed a point where we don't have any unmapped brick
  533 + // with a higher hit ratio then the lowest mapped brick. We're done.
  534 + break;
  535 +
  536 + LockableWindow window = mappedBrick.getWindow();
  537 + if (window.writeOutAndCloseIfFree( readOnly ) )
496 538 {
497   - BrickElement mappedBrick = mappedBricks.get( mappedIndex++ );
498   - BrickElement unmappedBrick = unmappedBricks.get( unmappedIndex-- );
499   - if ( mappedBrick.getHit() >= unmappedBrick.getHit() )
500   - // We've passed a point where we don't have any unmapped brick
501   - // with a higher hit ratio then the lowest mapped brick. We're done.
502   - break;
503   -
504   - LockableWindow window = mappedBrick.getWindow();
505   - if (window.writeOutAndCloseIfFree( readOnly ) )
506   - {
507   - mappedBrick.setWindow( null );
508   - memUsed -= brickSize;
509   - if ( allocateNewWindow( unmappedBrick ) )
510   - switches++;
511   - }
  539 + mappedBrick.setWindow( null );
  540 + memUsed -= brickSize;
  541 + if ( allocateNewWindow( unmappedBrick ) )
  542 + switches++;
512 543 }
513 544 }
514 545 }
@@ -644,8 +675,9 @@ private void logWarn( String logMessage, Throwable cause )
644 675
645 676 WindowPoolStats getStats()
646 677 {
  678 + int avgRefreshTime = refreshes.get() == 0 ? 0 : (int)(refreshTime.get()/refreshes.get());
647 679 return new WindowPoolStats( storeName, availableMem, memUsed, brickCount,
648   - brickSize, hit, miss, ooe );
  680 + brickSize, hit, miss, ooe, switches, avgRefreshTime, refreshes.get(), avertedRefreshes.get() );
649 681 }
650 682
651 683 private static class BrickElement
62 kernel/src/main/java/org/neo4j/kernel/impl/nioneo/store/WindowPoolStats.java
@@ -19,6 +19,8 @@
19 19 */
20 20 package org.neo4j.kernel.impl.nioneo.store;
21 21
  22 +import java.io.File;
  23 +
22 24 public class WindowPoolStats
23 25 {
24 26 private final String name;
@@ -32,11 +34,17 @@
32 34 private final int hitCount;
33 35 private final int missCount;
34 36 private final int oomCount;
  37 +
  38 + private final int switchCount;
  39 + private final int avgRefreshTime;
  40 + private final int refreshCount;
  41 + private final int avertedRefreshCount;
35 42
36 43 public WindowPoolStats( String name, long memAvail, long memUsed, int windowCount,
37   - int windowSize, int hitCount, int missCount, int oomCount )
  44 + int windowSize, int hitCount, int missCount, int oomCount, int switchCount, int avgRefreshTime,
  45 + int refreshCount, int avertedRefreshCount )
38 46 {
39   - this.name = name;
  47 + this.name = extractName( name );
40 48 this.memAvail = memAvail;
41 49 this.memUsed = memUsed;
42 50 this.windowCount = windowCount;
@@ -44,8 +52,17 @@ public WindowPoolStats( String name, long memAvail, long memUsed, int windowCoun
44 52 this.hitCount = hitCount;
45 53 this.missCount = missCount;
46 54 this.oomCount = oomCount;
  55 + this.switchCount = switchCount;
  56 + this.avgRefreshTime = avgRefreshTime;
  57 + this.refreshCount = refreshCount;
  58 + this.avertedRefreshCount = avertedRefreshCount;
47 59 }
48 60
  61 + private String extractName( String name )
  62 + {
  63 + return new File( name ).getName();
  64 + }
  65 +
49 66 public String getName()
50 67 {
51 68 return name;
@@ -86,16 +103,41 @@ public int getOomCount()
86 103 return oomCount;
87 104 }
88 105
  106 + public int getSwitchCount()
  107 + {
  108 + return switchCount;
  109 + }
  110 +
  111 + public int getAvgRefreshTime()
  112 + {
  113 + return avgRefreshTime;
  114 + }
  115 +
  116 + public int getRefreshCount()
  117 + {
  118 + return refreshCount;
  119 + }
  120 +
  121 + public int getAvertedRefreshCount()
  122 + {
  123 + return avertedRefreshCount;
  124 + }
  125 +
89 126 @Override
90 127 public String toString()
91 128 {
92   - return "WindowPoolStats[" +
93   - "mem available:" + memAvail + " " +
94   - "mem used:" + memUsed + " " +
95   - "windows:" + windowCount + " " +
96   - "win size:" + windowSize + " " +
97   - "hits:" + hitCount + " " +
98   - "misses:" + missCount + " " +
99   - "ooms:" + oomCount + "]";
  129 + return "WindowPoolStats['" + name + "', " +
  130 + "memAvail:" + memAvail + ", " +
  131 + "memUsed:" + memUsed + ", " +
  132 + "windowCount:" + windowCount + ", " +
  133 + "windowSize:" + windowSize + ", " +
  134 + "hitCount:" + hitCount + ", " +
  135 + "missCount:" + missCount + ", " +
  136 + "oomCount:" + oomCount + ", " +
  137 + "switchCount:" + switchCount + ", " +
  138 + "avgRefreshTime:" + avgRefreshTime + ", " +
  139 + "refreshCount:" + refreshCount + ", " +
  140 + "avertedRefreshCount:" + avertedRefreshCount +
  141 + "]";
100 142 }
101 143 }

0 comments on commit 21e9540

Please sign in to comment.
Something went wrong with that request. Please try again.