Improve memory fetching in Linux

agent/src/main/java/blasd/apex/core/agent/InstrumentationAgent.java

@@ -69,9 +69,7 @@ public static synchronized Optional<Instrumentation> getInstrumentation() {
 			} else {
 				BYTE_BUDDY_IS_INSTALLED.set(true);
 
-				final Accessor singleAttempt = ByteBuddyAgent.AttachmentProvider.DEFAULT.attempt();
-				DEFAULT_ATTEMPT.set(singleAttempt);
-
+				final Accessor singleAttempt = safeGetDefaultAttempt();
 				if (singleAttempt.isAvailable()) {
 					return Optional.of(ByteBuddyAgent.install(new AttachmentProvider() {
 
@@ -90,4 +88,15 @@ public Accessor attempt() {
 			return Optional.absent();
 		}
 	}
+
+	/**
+	 * Also referred by {blasd.apex.core.agent.VirtualMachineWithoutToolsJar#findVirtualMachineClass()}
+	 */
+	synchronized static Accessor safeGetDefaultAttempt() {
+		if (DEFAULT_ATTEMPT.get() == null) {
+			final Accessor singleAttempt = ByteBuddyAgent.AttachmentProvider.DEFAULT.attempt();
+			DEFAULT_ATTEMPT.compareAndSet(null, singleAttempt);
+		}
+		return DEFAULT_ATTEMPT.get();
+	}
 }

agent/src/main/java/blasd/apex/core/agent/VirtualMachineWithoutToolsJar.java

@@ -42,7 +42,6 @@
 import com.google.common.base.Optional;
 import com.google.common.collect.Sets;
 
-import net.bytebuddy.agent.ByteBuddyAgent;
 import net.bytebuddy.agent.ByteBuddyAgent.AttachmentProvider.Accessor;
 
 /**
@@ -179,7 +178,7 @@ public static synchronized Optional<Object> getUnsafeJvmVirtualMachine() throws
 	public static synchronized Optional<? extends Class<?>> findVirtualMachineClass() {
 		if (JVM_VIRTUAL_MACHINE_CLASS.get() == null) {
 			try {
-				Accessor attempt = InstrumentationAgent.DEFAULT_ATTEMPT.get();
+				Accessor attempt = InstrumentationAgent.safeGetDefaultAttempt();
 				if (attempt.isAvailable()) {
 					JVM_VIRTUAL_MACHINE_CLASS.set(attempt.getVirtualMachineType());
 				}

agent/src/main/java/org/ehcache/sizeof/impl/AgentLoaderApexSpy.java

@@ -24,18 +24,13 @@
 
 import java.lang.instrument.Instrumentation;
 import java.lang.reflect.Field;
-import java.lang.reflect.Method;
 import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicReference;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import com.google.common.base.Optional;
 
-import net.bytebuddy.agent.ByteBuddyAgent;
-import net.bytebuddy.agent.ByteBuddyAgent.AttachmentProvider.Accessor;
-
 /**
  * Introspect org.ehcache.sizeof.impl.AgentLoader to retrieve a reference to Instrumentation
  * 

java/src/main/java/blasd/apex/core/memory/ApexMemoryHelper.java

@@ -281,11 +281,22 @@ public static long memoryAsLong(String targetMax) {
 		String digits;
 		long multiplier;
 
-		char lastChar = targetMax.charAt(targetMax.length() - 1);
+		int lastDigit = targetMax.length();
+		char lastChar = targetMax.charAt(lastDigit - 1);
 		if (CharMatcher.javaDigit().matches(lastChar)) {
 			multiplier = 1L;
 			digits = targetMax;
 		} else {
+			lastDigit -= 1;
+
+			if (lastChar == 'b' || lastChar == 'B') {
+				if (targetMax.length() <= 1) {
+					return 0;
+				}
+				lastChar = targetMax.charAt(targetMax.length() - 2);
+				lastDigit -= 1;
+			}
+
 			if (lastChar == 'k' || lastChar == 'K') {
 				multiplier = IApexMemoryConstants.KB;
 			} else if (lastChar == 'm' || lastChar == 'M') {
@@ -297,7 +308,7 @@ public static long memoryAsLong(String targetMax) {
 						"Can not parse " + targetMax + ". It should end by a digit or one of 'k', 'm','g'");
 			}
 
-			digits = targetMax.substring(0, targetMax.length() - 1).trim();
+			digits = targetMax.substring(0, lastDigit).trim();
 		}
 
 		// Something like "123,456" or ""123 456""

java/src/main/java/blasd/apex/core/metrics/ApexMetricsTowerControl.java

@@ -362,10 +362,8 @@ public void onThrowable(Throwable t) {
 
 	protected void invalidateStartEvent(StartMetricEvent startEvent) {
 		if (activeTasks.getIfPresent(startEvent) == null) {
-			LOGGER.debug(
-					"An EndEvent has been submitted without its StartEvent having been registered"
-							+ ", or after having been already invalidated: {}",
-					startEvent);
+			LOGGER.debug("An EndEvent has been submitted without its StartEvent having been registered"
+					+ ", or after having been already invalidated: {}", startEvent);
 		} else {
 			invalidate(startEvent);
 		}

java/src/main/java/blasd/apex/core/util/ApexProcessHelper.java

@@ -99,7 +99,10 @@ public static OptionalLong getProcessResidentMemory(long pid) throws IOException
 			memoryBuilder = new ProcessBuilder("tasklist.exe", "/fi", "PID eq " + pidAsString, "/fo", "csv", "/nh");
 		} else {
 			// In heroku, no need for sudo
-			memoryBuilder = new ProcessBuilder("pmap", pidAsString);
+			// We need -X to be able to find the resident memory
+			// https://stackoverflow.com/questions/131303/how-to-measure-actual-memory-usage-of-an-application-or-process
+			// https://linux.die.net/man/1/pmap
+			memoryBuilder = new ProcessBuilder("pmap", "-x", pidAsString);
 		}
 
 		// SImplify output by redirecting all stream in the same place
@@ -238,11 +241,28 @@ public String getResult() {
 					throw new RuntimeException("Issue when extracting memory from '" + lastLine + "'", e);
 				}
 			} else {
-				// Last line is like ' total 65512K'
+				// With 'pmap -x':
+				// Address Kbytes RSS Dirty Mode Mapping
+				// ...
+				// ---------------- ------- ------- -------
+				//
+				// total kB 4824728 390624 377220
+
+				// With 'pmap', without '-x', Last line is like ' total 65512K'
 				lastLine = lastLine.trim();
 				if (lastLine.startsWith("total")) {
-					lastLine = lastLine.substring("total".length()).trim();
-					long memory = ApexMemoryHelper.memoryAsLong(lastLine);
+					lastLine = lastLine.substring("total".length()).replaceAll("\\s+", " ").trim();
+
+					int betweenTotalAndKbytes = lastLine.indexOf(' ');
+					String unit = lastLine.substring(0, betweenTotalAndKbytes);
+
+					int betweenKBytesAndRSS = lastLine.indexOf(' ', betweenTotalAndKbytes + 1);
+					// String kBytes = lastLine.substring(betweenKBytesAndRSS + 1, betweenKBytesAndRSS);
+
+					int betweenRSSAndDirty = lastLine.indexOf(' ', betweenKBytesAndRSS + 1);
+					String rss = lastLine.substring(betweenKBytesAndRSS + 1, betweenRSSAndDirty);
+
+					long memory = ApexMemoryHelper.memoryAsLong(rss + unit);
 					return OptionalLong.of(memory);
 				} else {
 					LOGGER.trace("Unexpected row: {}", lastLine);

java/src/test/java/blasd/apex/core/memory/TestApexMemoryHelper.java

@@ -65,6 +65,16 @@ public void testParseMemory() {
 		Assert.assertEquals(123 * IApexMemoryConstants.GB, ApexMemoryHelper.memoryAsLong("123g"));
 	}
 
+	@Test
+	public void testParseMemory_EndsKB() {
+		Assert.assertEquals(123 * IApexMemoryConstants.KB, ApexMemoryHelper.memoryAsLong("123kB"));
+	}
+
+	@Test
+	public void testParseMemory_Edge_B() {
+		Assert.assertEquals(0, ApexMemoryHelper.memoryAsLong("B"));
+	}
+
 	// Happens on vmmap|pmap. See ApexProcessHelper.getProcessResidentMemory(long)
 	@Test
 	public void testParseMemory_withDot() {

java/src/test/java/blasd/apex/core/util/TestApexProcessHelper.java

@@ -37,22 +37,19 @@
 public class TestApexProcessHelper {
 	@Test
 	public void testMemoryOnMac() throws IOException {
-		String macMemoryOutput = Arrays
-				.asList("mapped file 32.9M 10.7M 32K 0K 0K 0K 0K 139",
-						"shared memory 44K 44K 44K 0K 0K 0K 0K 6",
-						"=========== ======= ======== ===== ======= ======== ====== ===== =======",
-						"TOTAL 2.2G 538.2M 377.3M 0K 0K 16K 0K 845",
-						"TOTAL, minus reserved VM space 2.2G 538.2M 377.3M 0K 0K 16K 0K 845",
-						"",
-						"VIRTUAL RESIDENT DIRTY SWAPPED ALLOCATION BYTES DIRTY+SWAP REGION",
-						"MALLOC ZONE SIZE SIZE SIZE SIZE COUNT ALLOCATED FRAG SIZE % FRAG COUNT",
-						"=========== ======= ========= ========= ========= ========= ========= ========= ======",
-						"DefaultMallocZone_0x10b7b6000 203.0M 148.4M 87.4M 0K 167902 64.5M 22.9M 27% 19",
-						"GFXMallocZone_0x10b7e7000 0K 0K 0K 0K 0 0K 0K 0% 0",
-						"=========== ======= ========= ========= ========= ========= ========= ========= ======",
-						"TOTAL 203.0M 148.4M 87.4M 0K 167902 64.5M 22.9M 27% 19")
-				.stream()
-				.collect(Collectors.joining("\r"));
+		String macMemoryOutput = Arrays.asList("mapped file 32.9M 10.7M 32K 0K 0K 0K 0K 139",
+				"shared memory 44K 44K 44K 0K 0K 0K 0K 6",
+				"=========== ======= ======== ===== ======= ======== ====== ===== =======",
+				"TOTAL 2.2G 538.2M 377.3M 0K 0K 16K 0K 845",
+				"TOTAL, minus reserved VM space 2.2G 538.2M 377.3M 0K 0K 16K 0K 845",
+				"",
+				"VIRTUAL RESIDENT DIRTY SWAPPED ALLOCATION BYTES DIRTY+SWAP REGION",
+				"MALLOC ZONE SIZE SIZE SIZE SIZE COUNT ALLOCATED FRAG SIZE % FRAG COUNT",
+				"=========== ======= ========= ========= ========= ========= ========= ========= ======",
+				"DefaultMallocZone_0x10b7b6000 203.0M 148.4M 87.4M 0K 167902 64.5M 22.9M 27% 19",
+				"GFXMallocZone_0x10b7e7000 0K 0K 0K 0K 0 0K 0K 0% 0",
+				"=========== ======= ========= ========= ========= ========= ========= ========= ======",
+				"TOTAL 203.0M 148.4M 87.4M 0K 167902 64.5M 22.9M 27% 19").stream().collect(Collectors.joining("\r"));
 
 		long nbBytes = ApexProcessHelper
 				.extractMemory(ApexProcessHelper.OS_MARKER_MAC, new ByteArrayInputStream(macMemoryOutput.getBytes()))
@@ -72,7 +69,7 @@ public void testMemory_mac_multiplespaces() throws IOException {
 	}
 
 	@Test
-	public void testMemoryOnLinux() throws IOException {
+	public void testMemoryOnLinux_MissingDashX() throws IOException {
 		String macMemoryOutput = Arrays.asList(" total 65512K").stream().collect(Collectors.joining("\n"));
 
 		long nbBytes = ApexProcessHelper
@@ -81,6 +78,17 @@ public void testMemoryOnLinux() throws IOException {
 		Assert.assertEquals(65512 * IApexMemoryConstants.KB, nbBytes);
 	}
 
+	@Test
+	public void testMemoryOnLinux_WithDashX() throws IOException {
+		String macMemoryOutput =
+				Arrays.asList("total kB         4824728  390544  377152").stream().collect(Collectors.joining("\n"));
+
+		long nbBytes = ApexProcessHelper
+				.extractMemory(ApexProcessHelper.OS_MARKER_LINUX, new ByteArrayInputStream(macMemoryOutput.getBytes()))
+				.getAsLong();
+		Assert.assertEquals(65512 * IApexMemoryConstants.KB, nbBytes);
+	}
+
 	@Test
 	public void testMemoryOnWindows() throws IOException {
 		// "/fo csv"
@@ -89,22 +97,28 @@ public void testMemoryOnWindows() throws IOException {
 		// "/fo table"
 		// String windowsMemoryOutput = "chrome.exe 6740 Console 1 108,760 K";
 
-		long nbBytes = ApexProcessHelper.extractMemory(ApexProcessHelper.OS_MARKER_WINDOWS,
-				new ByteArrayInputStream(windowsMemoryOutput.getBytes())).getAsLong();
+		long nbBytes =
+				ApexProcessHelper
+						.extractMemory(ApexProcessHelper.OS_MARKER_WINDOWS,
+								new ByteArrayInputStream(windowsMemoryOutput.getBytes()))
+						.getAsLong();
 		Assert.assertEquals(107940 * IApexMemoryConstants.KB, nbBytes);
 	}
 
 	// French has no comma as thousands separator
 	@Test
-	public void testMemoryOnWindows_FranchLocal() throws IOException {
+	public void testMemoryOnWindows_FrenchLocal() throws IOException {
 		// "/fo csv"
 		String windowsMemoryOutput = "\"chrome.exe\",\"6740\",\"Console\",\"1\",\"78 332 K\"";
 
 		// "/fo table"
 		// String windowsMemoryOutput = "chrome.exe 6740 Console 1 108,760 K";
 
-		long nbBytes = ApexProcessHelper.extractMemory(ApexProcessHelper.OS_MARKER_WINDOWS,
-				new ByteArrayInputStream(windowsMemoryOutput.getBytes())).getAsLong();
+		long nbBytes =
+				ApexProcessHelper
+						.extractMemory(ApexProcessHelper.OS_MARKER_WINDOWS,
+								new ByteArrayInputStream(windowsMemoryOutput.getBytes()))
+						.getAsLong();
 		Assert.assertEquals(78332 * IApexMemoryConstants.KB, nbBytes);
 	}