Skip to content

Commit

Permalink
ftrace: improve binary parser speed by optimizing I/O
Browse files Browse the repository at this point in the history 
* mmap the whole file for faster parsing
* re-use the same mapping to read events

Signed-off-by: Fabrizio Iannetti <fabrizio.iannetti@gmail.com>
  • Loading branch information
@fabrizioiannetti @MatthewKhouzam
fabrizioiannetti authored and MatthewKhouzam committed Jun 18, 2024
1 parent bcb1108 commit 286f29e
Show file tree
Hide file tree
Showing 14 changed files with 327 additions and 321 deletions.
53 changes: 26 additions & 27 deletions ...se/tracecompass/incubator/ftrace/core/tests/binary/buffer/BinaryFTraceByteBufferTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,38 +45,37 @@ public void testOffsetCounter() throws IOException {
String traceLocation = FTraceUtils.getTraceAbsolutePath(FtraceTestTrace.TEST_2_6_MULTIPLE_CPUS);
long currentOffset = 0;

try (BinaryFTraceByteBuffer buffer = new BinaryFTraceByteBuffer(traceLocation)) {
// Read the magic values
buffer.getNextBytes(BinaryFTraceHeaderElementSize.MAGIC_VALUE);
currentOffset += BinaryFTraceHeaderElementSize.MAGIC_VALUE;
assertEquals(buffer.getCurrentOffset(), currentOffset);
BinaryFTraceByteBuffer buffer = new BinaryFTraceByteBuffer(traceLocation);
// Read the magic values
buffer.getNextBytes(BinaryFTraceHeaderElementSize.MAGIC_VALUE);
currentOffset += BinaryFTraceHeaderElementSize.MAGIC_VALUE;
assertEquals(buffer.getCurrentOffset(), currentOffset);

String tracingString = buffer.getNextBytesAsString(BinaryFTraceHeaderElementSize.TRACING_STRING);
currentOffset += BinaryFTraceHeaderElementSize.TRACING_STRING;
assertEquals(tracingString, TRACING);
assertEquals(buffer.getCurrentOffset(), currentOffset);
String tracingString = buffer.getNextBytesAsString(BinaryFTraceHeaderElementSize.TRACING_STRING);
currentOffset += BinaryFTraceHeaderElementSize.TRACING_STRING;
assertEquals(tracingString, TRACING);
assertEquals(buffer.getCurrentOffset(), currentOffset);

String versionString = buffer.getNextString();
currentOffset += (BinaryFTraceHeaderElementSize.FTRACE_VERSION + BinaryFTraceHeaderElementSize.STRING_TERMINATOR);
assertEquals(versionString, VERSION);
assertEquals(buffer.getCurrentOffset(), currentOffset);
String versionString = buffer.getNextString();
currentOffset += (BinaryFTraceHeaderElementSize.FTRACE_VERSION + BinaryFTraceHeaderElementSize.STRING_TERMINATOR);
assertEquals(versionString, VERSION);
assertEquals(buffer.getCurrentOffset(), currentOffset);

// Read various data types and validate offset changes
buffer.getNextShort();
currentOffset += BinaryFTraceDataType.SHORT.getSize();
assertEquals(buffer.getCurrentOffset(), currentOffset);
// Read various data types and validate offset changes
buffer.getNextShort();
currentOffset += BinaryFTraceDataType.SHORT.getSize();
assertEquals(buffer.getCurrentOffset(), currentOffset);

buffer.getNextInt();
currentOffset += BinaryFTraceDataType.INT.getSize();
assertEquals(buffer.getCurrentOffset(), currentOffset);
buffer.getNextInt();
currentOffset += BinaryFTraceDataType.INT.getSize();
assertEquals(buffer.getCurrentOffset(), currentOffset);

buffer.getNextLong();
currentOffset += BinaryFTraceDataType.LONG.getSize();
assertEquals(buffer.getCurrentOffset(), currentOffset);
buffer.getNextLong();
currentOffset += BinaryFTraceDataType.LONG.getSize();
assertEquals(buffer.getCurrentOffset(), currentOffset);

buffer.movePointerToOffset(RANDOM_OFFSET);
currentOffset = RANDOM_OFFSET;
assertEquals(buffer.getCurrentOffset(), currentOffset);
}
buffer.movePointerToOffset(RANDOM_OFFSET);
currentOffset = RANDOM_OFFSET;
assertEquals(buffer.getCurrentOffset(), currentOffset);
}
}
32 changes: 20 additions & 12 deletions ...pse/tracecompass/incubator/internal/ftrace/core/binary/header/BinaryFTraceHeaderInfo.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,9 @@
import java.util.Map;
import java.util.Optional;

import org.eclipse.tracecompass.incubator.internal.ftrace.core.binary.parser.BinaryFTraceByteBuffer;
import org.eclipse.tracecompass.incubator.internal.ftrace.core.binary.parser.BinaryFTraceFileMapping;

/**
* A representation of all header information required for parsing the events in
* the FTrace file.
Expand Down Expand Up @@ -46,12 +49,15 @@ public class BinaryFTraceHeaderInfo {

private List<BinaryFTraceFileCPU> cpus;

private BinaryFTraceFileMapping fTraceMapping;

/**
* Constructor
*
* @param builder The builder used to construct the trace header
* @param traceMapping the file memory mapping used to read data from the file
*/
private BinaryFTraceHeaderInfo(BinaryFTraceHeaderInfoBuilder builder) {
private BinaryFTraceHeaderInfo(BinaryFTraceHeaderInfoBuilder builder, BinaryFTraceFileMapping traceMapping) {
fFilePath = builder.fBuilderFilePath;
fVersion = builder.fBuilderVersion;
fEndianess = builder.fBuilderEndianess; // Because Java by default is
Expand All @@ -72,25 +78,26 @@ private BinaryFTraceHeaderInfo(BinaryFTraceHeaderInfoBuilder builder) {
fOptions = builder.fBuilderOptions;
fEventCommonFields = builder.fBuilderEventCommonFields;
cpus = builder.fBuilderCpus;
fTraceMapping = traceMapping;
fTraceMapping.order(fEndianess);
}

/**
* Get the file path to the trace file.
* Get the trace contents as a ByteBuffer.
*
* @return The file path of the trace file.
* @return the trace contents as a ByteBuffer
*/
public String getFilePath() {
return fFilePath;
public BinaryFTraceByteBuffer getMappedBuffer() {
return new BinaryFTraceByteBuffer(fTraceMapping);
}

/**
* Set the file path to the trace file.
* Get the file path to the trace file.
*
* @param filePath
* The file path of the trace file.
* @return The file path of the trace file.
*/
public void setFilePath(String filePath) {
this.fFilePath = filePath;
public String getFilePath() {
return fFilePath;
}

/**
Expand Down Expand Up @@ -612,11 +619,12 @@ public BinaryFTraceHeaderInfoBuilder cpus(List<BinaryFTraceFileCPU> builderCpus)
* Build an immutable {@link BinaryFTraceHeaderInfo} using the current
* state of the builder.
*
* @param traceMapping the memory mapped file buffer to be used for reading data
* @return A {@link BinaryFTraceHeaderInfo} object.
*/
public BinaryFTraceHeaderInfo build() {
public BinaryFTraceHeaderInfo build(BinaryFTraceFileMapping traceMapping) {
fBuilderEventCommonFields = getCommonFields();
return new BinaryFTraceHeaderInfo(this);
return new BinaryFTraceHeaderInfo(this, traceMapping);
}

private Map<String, BinaryFTraceFormatField> getCommonFields() {
Expand Down
93 changes: 30 additions & 63 deletions ...cecompass/incubator/internal/ftrace/core/binary/iterator/BinaryFTraceCPUPageIterator.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -64,10 +64,8 @@ public class BinaryFTraceCPUPageIterator implements Closeable {
* The CPU data page to read
* @param fileHeader
* The FTrace header information object
* @throws IOException
* When there is an error without opening/reading from the file
*/
public BinaryFTraceCPUPageIterator(@NonNull BinaryFTraceCPUDataPage page, @NonNull BinaryFTraceHeaderInfo fileHeader) throws IOException {
public BinaryFTraceCPUPageIterator(@NonNull BinaryFTraceCPUDataPage page, @NonNull BinaryFTraceHeaderInfo fileHeader) {
// Initialize buffer data
fFileHeader = fileHeader;
this.fPage = page;
Expand Down Expand Up @@ -103,11 +101,7 @@ public long getCurrentTimeStamp() {
*/
@Override
public void close() throws IOException {
BinaryFTraceByteBuffer buffer = fBuffer;
if (buffer != null) {
buffer.close();
fBuffer = null;
}
fBuffer = null;
}

/**
Expand Down Expand Up @@ -152,10 +146,8 @@ public BinaryFTraceResponse readNextEvent() {
/**
* Read the time stamp of the next event only (without reading the actual
* event)
*
* @throws IOException
*/
private void readNextEventTimeStamp() throws IOException {
private void readNextEventTimeStamp() {
boolean readSuccess = readNextEventHeader();

/**
Expand Down Expand Up @@ -205,9 +197,8 @@ private void readNextEventTimeStamp() throws IOException {
* increment the offset
*
* @return true if the event is read sucessfully
* @throws IOException
*/
private boolean readNextEventHeader() throws IOException {
private boolean readNextEventHeader() {
if (!hasNext()) {
return false;
}
Expand All @@ -232,11 +223,8 @@ private boolean readNextEventHeader() throws IOException {
*
* @return -1 if the there is no next event, else return the event type
* length.
* @throws IOException
* if there is an error reading the next event, likely because
* of buffer overflow caused a logical error during reading.
*/
private int peekNextEventType() throws IOException {
private int peekNextEventType() {
if (!hasNext()) {
return -1;
}
Expand All @@ -254,13 +242,8 @@ private int peekNextEventType() throws IOException {
/**
* Update the time stamp based on the state of the current event header and
* increment the offset if needed
*
* @throws IOException
* IOException if there is an error reading the next event,
* likely because of buffer overflow caused a logical error
* during reading.
*/
private void updateTimeStamp() throws IOException {
private void updateTimeStamp() {
if (fCurrentTypeLen <= fFileHeader.getHeaderEventInfo().getDataMaxTypeLen()) {
fCurrentTimeStamp += fCurrentTimeDelta;
} else {
Expand Down Expand Up @@ -289,11 +272,8 @@ private void skip(int bytesToSkip) {
* Get the pay load size of the event according to its type length
*
* @return The size of the payload of the event
* @throws IOException
* if there is an error reading the next event, likely because
* of buffer overflow caused a logical error during reading.
*/
private int getCurrentEventPayloadSize() throws IOException {
private int getCurrentEventPayloadSize() {

int payloadSize = 0;
if (fCurrentTypeLen == fFileHeader.getHeaderEventInfo().getCustomLengthEventTypeLen()) {
Expand Down Expand Up @@ -326,39 +306,34 @@ public BinaryFTraceCPUDataPage getPage() {
* Lazily read the next event using the event definition information.
*
* @return A BinaryFTraceEvent at the current event definition.
* @throws IOException
* if there is an error reading the next event, likely because
* of buffer overflow caused a logical error during reading.
*/
public @Nullable BinaryFTraceEvent getCurrentEvent() throws IOException {
public @Nullable BinaryFTraceEvent getCurrentEvent() {
// Make a local copy to prevent multithreading null check
BinaryFTraceEventDefinition eventDef = fEventDef;

if (eventDef != null) {
// We use a new buffer to read the current event
try (BinaryFTraceByteBuffer tempBuffer = new BinaryFTraceByteBuffer(fFileHeader.getFilePath());) {
tempBuffer.setByteOrder(fFileHeader.getEndianess());
tempBuffer.movePointerToOffset(eventDef.getPayloadOffset());

byte[] data = tempBuffer.getNextBytes(eventDef.getPayloadSize());
BinaryFTraceEventFormat eventFormat = fDataParser.getEventFormat(data);
if (eventFormat == null) {
return null;
}
BinaryFTraceByteBuffer tempBuffer = fFileHeader.getMappedBuffer();
tempBuffer.movePointerToOffset(eventDef.getPayloadOffset());

Map<String, Object> properties;
if (eventDef.getPayloadSize() > 0) {
properties = fDataParser.parseEventData(eventFormat, data);
} else {
properties = new HashMap<>();
}
byte[] data = tempBuffer.getNextBytes(eventDef.getPayloadSize());
BinaryFTraceEventFormat eventFormat = fDataParser.getEventFormat(data);
if (eventFormat == null) {
return null;
}

BinaryFTraceEvent event = new BinaryFTraceEvent(fCurrentTimeStamp,
properties,
eventFormat.getEventName(),
fPage.getCpu());
return event;
Map<String, Object> properties;
if (eventDef.getPayloadSize() > 0) {
properties = fDataParser.parseEventData(eventFormat, data);
} else {
properties = new HashMap<>();
}

BinaryFTraceEvent event = new BinaryFTraceEvent(fCurrentTimeStamp,
properties,
eventFormat.getEventName(),
fPage.getCpu());
return event;
}

return null;
Expand All @@ -370,11 +345,8 @@ public BinaryFTraceCPUDataPage getPage() {
* @param timeStampToSeekTo
* The timestamp to seek to
* @return true if the seek was successful
* @throws IOException
* if there is an error reading the next event, likely because
* that the iterator has reached the end.
*/
public boolean seek(long timeStampToSeekTo) throws IOException {
public boolean seek(long timeStampToSeekTo) {
// Reset the iterator
initializeIterator();

Expand Down Expand Up @@ -403,12 +375,8 @@ public boolean seek(long timeStampToSeekTo) throws IOException {
* Check if there are more stuff to read in this iterator
*
* @return true if there are more events to read in this page
*
* @throws IOException
* if there is an error reading the next event, likely because
* that the iterator has reached the end.
*/
public boolean hasNext() throws IOException {
public boolean hasNext() {
boolean ret = true;

BinaryFTraceByteBuffer buffer = fBuffer;
Expand Down Expand Up @@ -454,7 +422,7 @@ public boolean hasNext() throws IOException {
return ret;
}

private void initializeIterator() throws IOException {
private void initializeIterator() {
fCurrentOffset = fPage.getDataStartingOffset();
fCurrentTimeStamp = fPage.getTimeStamp();
fCurrentTypeLen = -1;
Expand All @@ -466,8 +434,7 @@ private void initializeIterator() throws IOException {
fBuffer.movePointerToOffset(fPage.getDataStartingOffset());
} else {
// Prepare the buffer
BinaryFTraceByteBuffer newBuffer = new BinaryFTraceByteBuffer(fFileHeader.getFilePath());
newBuffer.setByteOrder(fFileHeader.getEndianess());
BinaryFTraceByteBuffer newBuffer = fFileHeader.getMappedBuffer();
newBuffer.movePointerToOffset(fPage.getDataStartingOffset());

fBuffer = newBuffer;
Expand Down
5 changes: 1 addition & 4 deletions ...ompass/incubator/internal/ftrace/core/binary/iterator/BinaryFTraceCPUSectionIterator.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -140,11 +140,8 @@ public void close() throws IOException {
* Get the current event of this iterator lazily.
*
* @return The current event as a BinaryFTrace event.
* @throws IOException
* if there is an error reading the event, likely because of
* byte buffer failure.
*/
public @Nullable BinaryFTraceEvent getCurrentEvent() throws IOException {
public @Nullable BinaryFTraceEvent getCurrentEvent() {
BinaryFTraceCPUPageIterator iter = fCurrPageIterator;
if (iter != null) {
return iter.getCurrentEvent();
Expand Down

0 comments on commit 286f29e

Please sign in to comment.