Streamline bloat reports, part 1 (#33642)

* Streamline bloat reports, part 1

- Remove ‘Increases’ and ‘Decreases’ tables from the GitHub comment
  reports, leaving only the full table and the large-increase
  highlights.

- Add region groups to platform configurations. Reports will switch
  to summary flash and RAM totals rather than reporting each section.
  Some platform report configurations did not already indicate which
  section(s) belong in which reportable region.

- Add region sizes to size report artifacts.

- Record area type (section, region, read/write) in the size database.

A followup will remove sizes other than regions from the GitHub comment
reports.

* Restyled by autopep8

---------

Co-authored-by: Restyled.io <commits@restyled.io>

scripts/tools/memory/gh_report.py

@@ -316,9 +316,6 @@ def format(config: Config, df: pd.DataFrame):
             threshold_df = df[df['% change'] > threshold]
             if threshold_df.empty:
                 threshold_df = None
-        decrease_df = df[df['change'] < 0]
-        if decrease_df.empty:
-            decrease_df = None
 
         with io.StringIO() as md:
             md.write(df.attrs['title'])
@@ -329,22 +326,6 @@ def format(config: Config, df: pd.DataFrame):
                 md.write('<!--ghr-report:threshold-->\n\n')
                 V1Comment.write_df(config, threshold_df, md)
 
-            if increase_df is not None:
-                summary = V1Comment.summary(increase_df)
-                md.write('<details>\n')
-                md.write(f'<summary>Increases ({summary})</summary>\n')
-                md.write('<!--ghr-report:increases-->\n\n')
-                V1Comment.write_df(config, increase_df, md)
-                md.write('</details>\n\n')
-
-            if decrease_df is not None:
-                summary = V1Comment.summary(decrease_df)
-                md.write('<details>\n')
-                md.write(f'<summary>Decreases ({summary})</summary>\n')
-                md.write('<!--ghr-report:decreases-->\n\n')
-                V1Comment.write_df(config, decrease_df, md)
-                md.write('</details>\n\n')
-
             summary = V1Comment.summary(df)
             md.write('<details>\n')
             md.write(f'<summary>Full report ({summary})</summary>\n')

scripts/tools/memory/gh_sizes.py

@@ -183,6 +183,18 @@ def main(argv):
             if value := config[key]:
                 config.putl(['output', 'metadata', key], value)
 
+        # In case there is no platform configuration file or it does not define regions,
+        # try to find reasonable groups.
+        if not config.get('region.sections'):
+            sections = {'FLASH': [], 'RAM': []}
+            for section in config.get('section.select'):
+                print('section:', section)
+                for substring, region in [('text', 'FLASH'), ('rodata', 'FLASH'), ('data', 'RAM'), ('bss', 'RAM')]:
+                    if substring in section:
+                        sections[region].append(section)
+                        break
+            config.put('region.sections', sections)
+
         collected: DFs = memdf.collect.collect_files(config, [binary])
 
         # Aggregate loaded segments, by writable (RAM) or not (flash).
@@ -205,9 +217,13 @@ def main(argv):
                                     'wr']].sort_values(by='section')
         section_summary.attrs['name'] = "section"
 
+        region_summary = memdf.select.groupby(config, collected['section'], 'region')
+        region_summary.attrs['name'] = "region"
+
         summaries = {
             'section': section_summary,
             'memory': segment_summary,
+            'region': region_summary,
         }
 
         # Write configured (json) report to the output file.

scripts/tools/memory/memdf/sizedb.py

@@ -61,11 +61,12 @@ class SizeDatabase(memdf.util.sqlite.Database):
             UNIQUE(thing_id, hash, parent, pr, time, artifact)
         )
         """, """
-        -- A ‘size’ entry gives the size of a section for a particular build.
+        -- A ‘size’ entry gives the size of an area for a particular build.
         CREATE TABLE IF NOT EXISTS size (
             build_id    INTEGER REFERENCES build(id),
-            name        TEXT NOT NULL,      -- Section name
-            size        INTEGER NOT NULL,   -- Section size in bytes
+            kind        TEXT NOT NULL,      -- Area kind
+            name        TEXT NOT NULL,      -- Area name
+            size        INTEGER NOT NULL,   -- Size in bytes
             PRIMARY KEY (build_id, name)
         )
         """
@@ -100,15 +101,20 @@ def add_sizes_from_json(self, s: Union[bytes, str], origin: Dict):
         r = origin.copy()
         r.update(json.loads(s))
         r['sizes'] = []
-        # Add section sizes.
-        for i in r['frames'].get('section', []):
-            r['sizes'].append({'name': i['section'], 'size': i['size']})
+        # Add section and region sizes.
+        for frame in ['section', 'region']:
+            for i in r['frames'].get(frame, []):
+                r['sizes'].append({
+                    'name': i[frame],
+                    'size': i['size'],
+                    'kind': frame
+                })
         # Add segment sizes.
         for i in r['frames'].get('wr', []):
             r['sizes'].append({
                 'name': ('(read only)', '(read/write)')[int(i['wr'])],
-                'size':
-                i['size']
+                'size': i['size'],
+                'kind': 'wr'
             })
         self.add_sizes(**r)
 

scripts/tools/memory/platform/bl602.cfg

@@ -30,11 +30,17 @@
     #         'end': [],
     #     }
     # },
-    # 'region': {
-    #     # Regions are sets of sections that can be used for aggregate reports.
-    #     'sections': {
-    #         'FLASH': [],
-    #         'RAM': []
-    #     }
-    # },
+    'region': {
+        # Regions are sets of sections that can be used for aggregate reports.
+        'sections': {
+            'FLASH': [
+                '.text',
+                '.rodata',
+            ],
+            'RAM': [
+                '.bss',
+                '.data',
+            ],
+        }
+    },
 }

scripts/tools/memory/platform/mbed.cfg

@@ -30,13 +30,17 @@
 #           'end': [],
 #       }
 #   },
-#   'region': {
-#       # Regions are sets of sections that can be used for aggregate reports.
-#       'sections': {
-#           'FLASH': [
-#           ],
-#           'RAM': [
-#           ]
-#       }
-#   },
+    'region': {
+        # Regions are sets of sections that can be used for aggregate reports.
+        'sections': {
+            'FLASH': [
+                '.text',
+                '.rodata',
+            ],
+            'RAM': [
+                '.bss',
+                '.data',
+            ],
+        }
+    },
 }

scripts/tools/memory/platform/openiotsdk.cfg

@@ -20,4 +20,17 @@
         # when operating by sections.
         'default': ['.text', '.data', '.bss', '.stack', '.heap'],
     },
+    'region': {
+        # Regions are sets of sections that can be used for aggregate reports.
+        'sections': {
+            'FLASH': [
+                '.text',
+                '.rodata',
+            ],
+            'RAM': [
+                '.bss',
+                '.data',
+            ],
+        }
+    },
 }

scripts/tools/memory/platform/p6.cfg

@@ -30,13 +30,17 @@
 #           'end': [],
 #       }
 #   },
-#   'region': {
-#       # Regions are sets of sections that can be used for aggregate reports.
-#       'sections': {
-#           'FLASH': [
-#           ],
-#           'RAM': [
-#           ]
-#       }
-#   },
+    'region': {
+        # Regions are sets of sections that can be used for aggregate reports.
+        'sections': {
+            'FLASH': [
+                '.text',
+                '.rodata',
+            ],
+            'RAM': [
+                '.bss',
+                '.data',
+            ],
+        }
+    },
 }

scripts/tools/memory/platform/telink.cfg

@@ -30,13 +30,19 @@
 #           'end': [],
 #       }
 #   },
-#   'region': {
-#       # Regions are sets of sections that can be used for aggregate reports.
-#       'sections': {
-#           'FLASH': [
-#           ],
-#           'RAM': [
-#           ]
-#       }
-#   },
+    'region': {
+        # Regions are sets of sections that can be used for aggregate reports.
+        'sections': {
+            'FLASH': [
+                'text',
+                'rodata',
+            ],
+            'RAM': [
+                'bss',
+                'data',
+                'noinit',
+                'ram_code',
+            ],
+        }
+    },
 }