From c3c1926bf813f8b3a50140a27f36248366f0c22d Mon Sep 17 00:00:00 2001 From: Timo Paulssen Date: Wed, 6 Sep 2017 13:30:53 +0200 Subject: [PATCH] write parts of types/static frames/strings incrementally so in case the program crashes before it can finish up, we can still recover everything we need. --- src/profiler/heapsnapshot.c | 61 +++++++++++++++++++++++++++++-------- src/profiler/heapsnapshot.h | 5 +++ 2 files changed, 54 insertions(+), 12 deletions(-) diff --git a/src/profiler/heapsnapshot.c b/src/profiler/heapsnapshot.c index 65f6aad773..cb83c6578b 100644 --- a/src/profiler/heapsnapshot.c +++ b/src/profiler/heapsnapshot.c @@ -671,10 +671,12 @@ void string_heap_to_filehandle(MVMThreadContext *tc, MVMHeapSnapshotCollection * /* Write out the number of strings we have and record the "header" size * in the index. */ - fwrite(&col->num_strings, sizeof(MVMuint64), 1, fh); + i = col->num_strings - col->strings_written; + + fwrite(&i, sizeof(MVMuint64), 1, fh); index->stringheap_size = sizeof(MVMuint64) + 4; - for (i = 0; i < col->num_strings; i++) { + for (i = col->strings_written; i < col->num_strings; i++) { char *str = col->strings[i]; MVMuint64 output_size = strlen(str); @@ -688,6 +690,8 @@ void string_heap_to_filehandle(MVMThreadContext *tc, MVMHeapSnapshotCollection * index->stringheap_size += sizeof(MVMuint64) + sizeof(MVMuint8) * output_size; } + + col->strings_written = col->num_strings; } /* The following functions all act the exact same way: @@ -695,7 +699,11 @@ void string_heap_to_filehandle(MVMThreadContext *tc, MVMHeapSnapshotCollection * * Write a little introductory text of 4 bytes for the parser to ensure * the index is correct, write the number of entries and the size of each entry * as 64bit integers, calculate the complete size for the index, and then - * just write out each entry */ + * just write out each entry + * + * We also write a partial table after every snapshot so that if the process + * crashes we still have a usable file. + */ void types_to_filehandle(MVMThreadContext *tc, MVMHeapSnapshotCollection *col) { MVMuint64 i; MVMHeapDumpIndex *index = col->index; @@ -703,18 +711,22 @@ void types_to_filehandle(MVMThreadContext *tc, MVMHeapSnapshotCollection *col) { fputs("type", fh); - fwrite(&col->num_types, sizeof(MVMuint64), 1, fh); + i = col->num_types - col->types_written; + + fwrite(&i, sizeof(MVMuint64), 1, fh); i = sizeof(MVMuint64) * 2; fwrite(&i, sizeof(MVMuint64), 1, fh); - index->types_size = sizeof(MVMuint64) * 2 + 4 + sizeof(MVMuint64) * 2 * col->num_types; + index->types_size = sizeof(MVMuint64) * 2 + 4 + sizeof(MVMuint64) * 2 * (col->num_types - col->types_written); - for (i = 0; i < col->num_types; i++) { + for (i = col->types_written; i < col->num_types; i++) { MVMHeapSnapshotType *t = &col->types[i]; fwrite(&t->repr_name, sizeof(MVMuint64), 1, fh); fwrite(&t->type_name, sizeof(MVMuint64), 1, fh); } + + col->types_written = col->num_types; } void static_frames_to_filehandle(MVMThreadContext *tc, MVMHeapSnapshotCollection *col) { MVMuint64 i; @@ -722,12 +734,15 @@ void static_frames_to_filehandle(MVMThreadContext *tc, MVMHeapSnapshotCollection FILE *fh = col->fh; fputs("fram", fh); - fwrite(&col->num_static_frames, sizeof(MVMuint64), 1, fh); + + i = col->num_static_frames - col->static_frames_written; + + fwrite(&i, sizeof(MVMuint64), 1, fh); i = sizeof(MVMuint64) * 4; fwrite(&i, sizeof(MVMuint64), 1, fh); - index->staticframes_size = sizeof(MVMuint64) * 2 + 4 + sizeof(MVMuint64) * 4 * col->num_static_frames; + index->staticframes_size = sizeof(MVMuint64) * 2 + 4 + sizeof(MVMuint64) * 4 * (col->num_static_frames - col->static_frames_written); - for (i = 0; i < col->num_static_frames; i++) { + for (i = col->static_frames_written; i < col->num_static_frames; i++) { MVMHeapSnapshotStaticFrame *sf = &col->static_frames[i]; fwrite(&sf->name, sizeof(MVMuint64), 1, fh); @@ -735,6 +750,8 @@ void static_frames_to_filehandle(MVMThreadContext *tc, MVMHeapSnapshotCollection fwrite(&sf->line, sizeof(MVMuint64), 1, fh); fwrite(&sf->file, sizeof(MVMuint64), 1, fh); } + + col->static_frames_written = col->num_static_frames; } /* The collectables table gets an entry in the additional "snapshot sizes @@ -749,12 +766,15 @@ void collectables_to_filehandle(MVMThreadContext *tc, MVMHeapSnapshotCollection MVMHeapSnapshot *s = col->snapshot; fputs("coll", fh); + fwrite(&s->num_collectables, sizeof(MVMuint64), 1, fh); i = sizeof(MVMuint16) * 2 + sizeof(MVMuint32) * 2 + sizeof(MVMuint64) * 2; fwrite(&i, sizeof(MVMuint64), 1, fh); entry->collectables_size += s->num_collectables * i + 4 + sizeof(MVMuint64) * 2; + fprintf(stderr, "writing collectables: %llx collectables of size %x each; collectables_size is %llx\n", s->num_collectables, i, entry->collectables_size); + for (i = 0; i < s->num_collectables; i++) { MVMHeapSnapshotCollectable *coll = &s->collectables[i]; fwrite(&coll->kind, sizeof(MVMuint16), 1, fh); @@ -787,7 +807,7 @@ void references_to_filehandle(MVMThreadContext *tc, MVMHeapSnapshotCollection *c fputs("refs", fh); fwrite(&s->num_references, sizeof(MVMuint64), 1, fh); - i = sizeof(MVMuint64) * 3; + i = sizeof(MVMuint64) * 2 + 1; fwrite(&i, sizeof(MVMuint64), 1, fh); entry->full_refs_size = 4 + sizeof(MVMuint64) * 2; @@ -839,9 +859,12 @@ void references_to_filehandle(MVMThreadContext *tc, MVMHeapSnapshotCollection *c fwrite(&index8, sizeof(MVMuint8), 1, fh); entry->full_refs_size += sizeof(MVMuint8) * 3 + 1; } - if (i == halfway) + if (i == halfway) { + fprintf(stderr, "refs: half-way point reached with %x entries at %llx\n", i, entry->full_refs_size); entry->refs_middlepoint = entry->full_refs_size; + } } + fprintf(stderr, "refs: done with %x entries at %llx\n", s->num_references, entry->full_refs_size); } void snapshot_to_filehandle(MVMThreadContext *tc, MVMHeapSnapshotCollection *col) { @@ -859,9 +882,18 @@ void snapshot_to_filehandle(MVMThreadContext *tc, MVMHeapSnapshotCollection *col entry->collectables_size = 0; entry->full_refs_size = 0; entry->refs_middlepoint = 0; + entry->incremental_data = 0; collectables_to_filehandle(tc, col, entry); references_to_filehandle(tc, col, entry); + + string_heap_to_filehandle(tc, col); + types_to_filehandle(tc, col); + static_frames_to_filehandle(tc, col); + + entry->incremental_data = index->stringheap_size + index->types_size + index->staticframes_size; + + fprintf(stderr, "incremental data added, it was %llx big in total\n", entry->incremental_data); } void index_to_filehandle(MVMThreadContext *tc, MVMHeapSnapshotCollection *col) { MVMHeapDumpIndex *index = col->index; @@ -874,6 +906,10 @@ void index_to_filehandle(MVMThreadContext *tc, MVMHeapSnapshotCollection *col) { fwrite(&index->snapshot_size_entries, sizeof(MVMuint64), 1, fh); } void finish_collection_to_filehandle(MVMThreadContext *tc, MVMHeapSnapshotCollection *col) { + col->strings_written = 0; + col->types_written = 0; + col->static_frames_written = 0; + string_heap_to_filehandle(tc, col); types_to_filehandle(tc, col); static_frames_to_filehandle(tc, col); @@ -890,6 +926,7 @@ void MVM_profile_heap_take_snapshot(MVMThreadContext *tc) { record_snapshot(tc, col, col->snapshot); snapshot_to_filehandle(tc, col); + fflush(col->fh); destroy_current_heap_snapshot(tc); col->snapshot_idx++; @@ -907,7 +944,7 @@ MVMObject * MVM_profile_heap_end(MVMThreadContext *tc) { dataset = tc->instance->VMNull; finish_collection_to_filehandle(tc, tc->instance->heap_snapshots); - destroy_heap_snapshot_collection(tc); fclose(col->fh); + destroy_heap_snapshot_collection(tc); return dataset; } diff --git a/src/profiler/heapsnapshot.h b/src/profiler/heapsnapshot.h index 2efb309c58..8f66e3a41e 100644 --- a/src/profiler/heapsnapshot.h +++ b/src/profiler/heapsnapshot.h @@ -2,6 +2,7 @@ struct MVMHeapDumpIndexSnapshotEntry { MVMuint64 collectables_size; MVMuint64 full_refs_size; MVMuint64 refs_middlepoint; + MVMuint64 incremental_data; }; struct MVMHeapDumpIndex { @@ -44,6 +45,10 @@ struct MVMHeapSnapshotCollection { MVMuint64 num_strings_free; MVMuint64 alloc_strings_free; + MVMuint64 types_written; + MVMuint64 static_frames_written; + MVMuint64 strings_written; + MVMHeapDumpIndex *index; /* The file handle we are outputting to */