Mas i1820 offlinedeserialisation cbo (#403)

* Log report GC Info by manifest level

* Hibernate on range query

If Block Index Cache is not full, and we're not yielding

* Spawn to deserialise blocks offline

Hypothesis is that the growth in the heap necessary due to continual term_to_binary calls to deserialise blocks is wasting memory - so do this memory-intensive task in a short-lived process.

* Start with hibernate_after option

* Always build BIC

Testing indicates that the BIC itself is not a primary memory issue - the primary issue is due to a lack of garbage collection and a growing heap.

This change enhances the patch to offline serialisation so that:
- get_sqn & get_kv are standardised to build the BIC, and hibernate when it is built.
- the offline PId is linked to crash this process on failure (as would happen now).

* Standardise spawning for check_block/3

Now deserialise in both parts of the code.

* Only spawn for check_block if cache not full

* Update following review

src/leveled_log.erl

@@ -168,6 +168,8 @@
             {info, <<"At level=~w file_count=~w avg_mem=~w file with most memory fn=~s p=~w mem=~w">>},
         pc024 =>
             {info, <<"Grooming compaction picked file with tomb_count=~w">>},
+        pc025 =>
+            {info, <<"At level=~w file_count=~w average words for heap_block_size=~w heap_size=~w recent_size=~w bin_vheap_size=~w">>},
         pm002 =>
             {info, <<"Completed dump of L0 cache to list of l0cache_size=~w">>},
         sst03 =>

src/leveled_pclerk.erl

@@ -200,11 +200,15 @@ handle_work(
             list(leveled_pmanifest:manifest_entry())}.
 merge(SrcLevel, Manifest, RootPath, OptsSST) ->
     case leveled_pmanifest:report_manifest_level(Manifest, SrcLevel + 1) of
-        {0, 0, undefined} ->
+        {0, 0, undefined, 0, 0, 0, 0} ->
             ok;
-        {FCnt, AvgMem, {MaxFN, MaxP, MaxMem}} ->
-            leveled_log:log(pc023,
-                            [SrcLevel + 1, FCnt, AvgMem, MaxFN, MaxP, MaxMem])
+        {FCnt, MnMem, {MaxFN, MaxP, MaxMem}, MnHBS, MnHS, MnLHS, MnBVHS} ->
+            leveled_log:log(
+                pc023,
+                [SrcLevel + 1, FCnt, MnMem, MaxFN, MaxP, MaxMem]),
+            leveled_log:log(
+                pc025,
+                [SrcLevel + 1, FCnt, MnHBS, MnHS, MnLHS, MnBVHS])
     end,
     SelectMethod =
         case leveled_rand:uniform(100) of

src/leveled_penciller.erl

@@ -498,7 +498,7 @@ pcl_workforclerk(Pid) ->
 -spec pcl_manifestchange(pid(), leveled_pmanifest:manifest()) -> ok.
 %% @doc
 %% Provide a manifest record (i.e. the output of the leveled_pmanifest module)
-%% that is required to beocme the new manifest.
+%% that is required to become the new manifest.
 pcl_manifestchange(Pid, Manifest) ->
     gen_server:cast(Pid, {manifest_change, Manifest}).
 

src/leveled_pmanifest.erl

@@ -266,11 +266,16 @@ remove_manifest(RootPath, GC_SQN) ->
         end.
 
 
--spec report_manifest_level(manifest(), non_neg_integer()) ->
-                            {non_neg_integer(),
-                                non_neg_integer(),
-                                {string(), pid(), non_neg_integer()} |
-                                    undefined}.
+-spec report_manifest_level(
+    manifest(), non_neg_integer()) ->
+        {non_neg_integer(),
+            non_neg_integer(),
+            {string(), pid(), non_neg_integer()} |
+                undefined,
+            non_neg_integer(),
+            non_neg_integer(),
+            non_neg_integer(),
+            non_neg_integer()}.
 %% @doc
 %% Report on a level in the manifest
 %% - How many files in the level
@@ -287,7 +292,7 @@ report_manifest_level(Manifest, LevelIdx) ->
                 {leveled_tree:tsize(Level), leveled_tree:to_list(Level)}
         end,
     AccMemFun = 
-        fun(MaybeME, {MemAcc, Max}) ->
+        fun(MaybeME, {MemAcc, Max, HBSAcc, HSAcc, LHSAcc, BVHSAcc}) ->
             ME = get_manifest_entry(MaybeME),
             P = ME#manifest_entry.owner,
             {memory, PM} = process_info(P, memory),
@@ -298,15 +303,26 @@ report_manifest_level(Manifest, LevelIdx) ->
                     _ ->
                         {ME#manifest_entry.filename, P, PM}
                 end,
-            {MemAcc + PM, UpdMax}
+            {garbage_collection_info, GCI} =
+                process_info(P, garbage_collection_info),
+            HBS = proplists:get_value(heap_block_size, GCI),
+            HS = proplists:get_value(heap_size, GCI),
+            LHS = proplists:get_value(recent_size, GCI),
+            BVHS = proplists:get_value(bin_vheap_size, GCI),
+            {MemAcc + PM, UpdMax,
+                HBSAcc + HBS, HSAcc + HS, LHSAcc + LHS, BVHSAcc + BVHS}
         end,
     case LevelSize of
         0 ->
-            {0, 0, undefined};
+            {0, 0, undefined, 0, 0, 0, 0};
         _ ->
-            {TotalMem, BiggestMem} =
-                lists:foldl(AccMemFun, {0, undefined}, LevelList),
-            {LevelSize, TotalMem div LevelSize, BiggestMem}
+            {TotalMem, BiggestMem, TotalHBS, TotalHS, TotalLHS, TotalBVBS} =
+                lists:foldl(AccMemFun, {0, undefined, 0, 0, 0, 0}, LevelList),
+            {LevelSize, TotalMem div LevelSize, BiggestMem,
+                TotalHBS div LevelSize,
+                TotalHS div LevelSize,
+                TotalLHS div LevelSize,
+                TotalBVBS div LevelSize}
     end.