Separate active/pending delete project status in inventory summary em…

…ail (#2132)

* + Add lifecycle status query function
+ Add hidden dataset summary function
+ Call above functions in get summary function and add to return dict

* + Add more params to test_inventory_summary_can_run_successfully function to attribute added functions in storage.py (hidden dataset and delete_pending/active project)

* + Formatting - have bracket and parenthesis on same line as code end

* + Add in SHOWN field for hidden resource type func
+ Add in DELETE PENDING field for lifecycleState func if not exist
+ Add _get_summary_details_data func
+ Add in summary_details_data param to _send_email func
+ Combine summary and details data as a param to _upload_to_gcs func
+ Add in separate section at bottom of inventory section for details
+ Add in Inventory Summary and Inventory Details headers
+ Return values and expected data incorporate details data in test

* - Take out delete/hidden description in get_summary as it no longer applies

* - Take out literal package
- Fix indentation issues

* + Add extra space

* + Change summary_details to details for better differentiation between summary and details data

* + indent line to fit PEP-8

* + Reformat docstrings to fit PEP 8
+ Refactor duplicate code into separate method

* + Change _transform_for_template to _transform_to_template
+ Add docstring to _transform_to_template

* - Take out None filter for lifecycleState query
+ Create separate variables for hidden/shown resource query for cleaner code
+ Add additional elif statement for resource type details for scalability
+ Modify expected_summary_data_send_email to incorp details section in test

* + Indent lines to fit PEP 8 style

* + Add more indentation for PEP 8 style

* + Add indent for PEP 8 style
+ Add @static_class to func

* + Add in if-else to factor in for either ACTIVE or DELETE PENDING
+ Change docstring description from list of dicts to just dict

* + Change sum to count in hidden_resource_type func to prevent None from appearing in count (instead 0 would be displayed)

* + Fix formatting of indent

google/cloud/forseti/common/email_templates/inventory_summary.jinja

@@ -20,6 +20,7 @@ limitations under the License.
     Inventory Index Id: {{ inventory_index_id }}<br>
     Inventory Timestamp: {{ timestamp }}<br>
     <br>
+    <h3> Inventory Summary </h3>
     <table>
         <col width="250">
         <col width="150">
@@ -34,5 +35,23 @@ limitations under the License.
         </tr>
         {% endfor %}
     </table>
+    <br>
+    <h3>Inventory Details </h3>
+    <table>
+        <col width="250">
+        <col width="150">
+        <tr>
+            <th align="left">Resource Name</th>
+            <th align="left">Count</th>
+        </tr>
+        <tr>
+            {% for data in details_data %}
+        </tr>
+        <tr>
+            <td>{{ data.resource_type }}</td>
+            <td>{{ data.count }}</td>
+        </tr>
+        {% endfor %}
+    </table>
 </body>
 </html>

google/cloud/forseti/notifier/notifiers/inventory_summary.py

@@ -100,12 +100,15 @@ def _upload_to_gcs(self, summary_data):
             LOGGER.exception('Unable to upload inventory summary in bucket %s:',
                              gcs_upload_path)
 
-    def _send_email(self, summary_data):
+    def _send_email(self, summary_data, details_data):
         """Send the email for inventory summary.
 
         Args:
             summary_data (list): Summary of inventory data as a list of dicts.
                 Example: [{resource_type, count}, {}, {}, ...]
+
+            details_data (list): Details of inventory data as a list of dicts.
+                Example: [[{resource_type, count}, {}, {}, ...]
         """
         LOGGER.debug('Sending inventory summary by email.')
 
@@ -127,7 +130,8 @@ def _send_email(self, summary_data):
             'inventory_summary.jinja',
             {'inventory_index_id': self.inventory_index_id,
              'timestamp': timestamp,
-             'summary_data': summary_data})
+             'summary_data': summary_data,
+             'details_data': details_data})
 
         try:
             email_util.send(
@@ -140,6 +144,23 @@ def _send_email(self, summary_data):
         except util_errors.EmailSendError:
             LOGGER.exception('Unable to send inventory summary email')
 
+    @staticmethod
+    def transform_to_template(data):
+        """Helper method to return sorted list of dicts.
+
+        Args:
+            data (dict): dictionary of resource_type: count pairs.
+
+        Returns:
+            list: Sorted data as a list of dicts.
+                Example: [{resource_type, count}, {}, {}, ...]
+
+        """
+        template_data = []
+        for key, value in data.iteritems():
+            template_data.append(dict(resource_type=key, count=value))
+        return sorted(template_data, key=lambda k: k['resource_type'])
+
     def _get_summary_data(self):
         """Get the summarized inventory data.
 
@@ -161,13 +182,33 @@ def _get_summary_data(self):
                              'index id: %s.', self.inventory_index_id)
                 raise util_errors.NoDataError
 
-            summary_data = []
-            for key, value in summary.iteritems():
-                summary_data.append(dict(resource_type=key, count=value))
-            summary_data = (
-                sorted(summary_data, key=lambda k: k['resource_type']))
+            summary_data = self.transform_to_template(summary)
             return summary_data
 
+    def _get_details_data(self):
+        """Get the detailed summarized inventory data.
+
+        Returns:
+            list: Summary details of sorted inventory data as a list of dicts.
+                Example: [{resource_type, count}, {}, {}, ...]
+
+        Raises:
+            NoDataError: If summary details data is not found.
+        """
+        LOGGER.debug('Getting inventory summary details data.')
+        with self.service_config.scoped_session() as session:
+            inventory_index = (
+                session.query(InventoryIndex).get(self.inventory_index_id))
+
+            details = inventory_index.get_details(session)
+            if not details:
+                LOGGER.error('No inventory summary detail data found for '
+                             'inventory index id: %s.', self.inventory_index_id)
+                raise util_errors.NoDataError
+
+            details_data = self.transform_to_template(details)
+            return details_data
+
     def run(self):
         """Generate inventory summary."""
         LOGGER.info('Running inventory summary notifier.')
@@ -194,16 +235,19 @@ def run(self):
 
         try:
             summary_data = self._get_summary_data()
+            details_data = self._get_details_data()
         except util_errors.NoDataError:
             LOGGER.exception('Inventory summary can not be created because '
                              'no summary data is found for index id: %s.',
                              self.inventory_index_id)
             return
 
         if is_gcs_summary_enabled:
-            self._upload_to_gcs(summary_data)
+            summary_and_details_data = sorted((summary_data + details_data),
+                                              key=lambda k: k['resource_type'])
+            self._upload_to_gcs(summary_and_details_data)
 
         if is_email_summary_enabled:
-            self._send_email(summary_data)
+            self._send_email(summary_data, details_data)
 
         LOGGER.info('Completed running inventory summary.')

google/cloud/forseti/services/inventory/storage.py

@@ -19,6 +19,7 @@
 
 from sqlalchemy import and_
 from sqlalchemy import BigInteger
+from sqlalchemy import case
 from sqlalchemy import Column
 from sqlalchemy import DateTime
 from sqlalchemy import Enum
@@ -154,6 +155,81 @@ def set_error(self, session, message):
         session.add(self)
         session.flush()
 
+    def get_lifecycle_state_details(self, session, resource_type_input):
+        """Count of lifecycle states of the specified resources.
+
+        Generate/return the count of lifecycle states (ACTIVE, DELETE_PENDING)
+        of the specific resource type input (project, folder) for this inventory
+        index.
+
+        Args:
+            session (object) : session object to work on.
+            resource_type_input (str) : resource type to get lifecycle states.
+
+        Returns:
+            dict: a (lifecycle state -> count) dictionary
+        """
+        resource_data = Inventory.resource_data
+
+        details = dict(
+            session.query(func.json_extract(resource_data, '$.lifecycleState'),
+                          func.count())
+            .filter(Inventory.inventory_index_id == self.id)
+            .filter(Inventory.category == 'resource')
+            .filter(Inventory.resource_type == resource_type_input)
+            .group_by(func.json_extract(resource_data, '$.lifecycleState'))
+            .all())
+
+        for key in details.keys():
+            new_key = key.replace('\"', '').replace('_', ' ')
+            new_key = ' - '.join([resource_type_input, new_key])
+            details[new_key] = details.pop(key)
+
+        if len(details) == 1:
+            if 'ACTIVE' in details.keys()[0]:
+                added_key_str = 'DELETE PENDING'
+            elif 'DELETE PENDING' in details.keys()[0]:
+                added_key_str = 'ACTIVE'
+            added_key = ' - '.join([resource_type_input, added_key_str])
+            details[added_key] = 0
+
+        return details
+
+    def get_hidden_resource_details(self, session, resource_type):
+        """Count of the hidden and shown specified resources.
+
+        Generate/return the count of hidden resources (e.g. dataset) for this
+        inventory index.
+
+        Args:
+            session (object) : session object to work on.
+            resource_type (str) : resource type to find details for.
+
+        Returns:
+            dict: a (hidden_resource -> count) dictionary
+        """
+        details = {}
+        resource_id = Inventory.resource_id
+        field_label_hidden = resource_type + ' - HIDDEN'
+        field_label_shown = resource_type + ' - SHOWN'
+
+        hidden_label = (
+            func.count(case([(resource_id.contains('%:~_%', escape='~'), 1)])))
+
+        shown_label = (
+            func.count(case([(~resource_id.contains('%:~_%', escape='~'), 1)])))
+
+        details_query = (
+            session.query(hidden_label, shown_label)
+            .filter(Inventory.inventory_index_id == self.id)
+            .filter(Inventory.category == 'resource')
+            .filter(Inventory.resource_type == resource_type).one())
+
+        details[field_label_hidden] = details_query[0]
+        details[field_label_shown] = details_query[1]
+
+        return details
+
     def get_summary(self, session):
         """Generate/return an inventory summary for this inventory index.
 
@@ -163,13 +239,49 @@ def get_summary(self, session):
         Returns:
             dict: a (resource type -> count) dictionary
         """
+
         resource_type = Inventory.resource_type
-        return dict(
+
+        summary = dict(
             session.query(resource_type, func.count(resource_type))
             .filter(Inventory.inventory_index_id == self.id)
             .filter(Inventory.category == 'resource')
             .group_by(resource_type).all())
 
+        return summary
+
+    def get_details(self, session):
+        """Generate/return inventory details for this inventory index.
+
+        Includes delete pending/active resource types and hidden/shown datasets.
+
+        Args:
+            session (object): session object to work on.
+
+        Returns:
+            dict: a (resource type -> count) dictionary
+        """
+        resource_types_with_lifecycle = ['folder', 'organization', 'project']
+        resource_types_hidden = ['dataset']
+
+        resource_types_with_details = {'lifecycle':
+                                       resource_types_with_lifecycle,
+                                       'hidden':
+                                       resource_types_hidden}
+
+        details = {}
+
+        for key, value in resource_types_with_details.items():
+            if key == 'lifecycle':
+                details_function = self.get_lifecycle_state_details
+            elif key == 'hidden':
+                details_function = self.get_hidden_resource_details
+            for resource in value:
+                resource_details = details_function(session, resource)
+                details.update(resource_details)
+
+        return details
+
 
 class Inventory(BASE):
     """Resource inventory table."""

tests/notifier/notifiers/inventory_summary_test.py

@@ -295,7 +295,18 @@ def test_inventory_summary_no_summary_data(self, mock_logger):
     def test_inventory_summary_can_run_successfully(self):
         mock_inventory_index = mock.MagicMock()
         mock_inventory_index.get_summary.return_value = {
-            'bucket': 2, 'object': 1, 'organization': 1, 'project': 2}
+            'bucket': 2,
+            'dataset': 4,
+            'folder': 1,
+            'object': 1,
+            'organization': 1,
+            'project': 2}
+
+        mock_inventory_index.get_details.return_value = {
+            'dataset - HIDDEN': 2,
+            'dataset - SHOWN': 2,
+            'project - ACTIVE': 1,
+            'project - DELETE PENDING': 1}
 
         mock_session = mock.MagicMock()
         mock_session.query.return_value.get.return_value = mock_inventory_index
@@ -317,21 +328,42 @@ def test_inventory_summary_can_run_successfully(self):
         notifier._send_email = mock.MagicMock()
         notifier.run()
 
-        expected_summary_data = [
+        expected_summary_data_upload_to_gcs = [
             {'count': 2, 'resource_type': 'bucket'},
+            {'count': 4, 'resource_type': 'dataset'},
+            {'count': 2, 'resource_type': 'dataset - HIDDEN'},
+            {'count': 2, 'resource_type': 'dataset - SHOWN'},
+            {'count': 1, 'resource_type': 'folder'},
             {'count': 1, 'resource_type': 'object'},
             {'count': 1, 'resource_type': 'organization'},
-            {'count': 2, 'resource_type': 'project'}]
+            {'count': 2, 'resource_type': 'project'},
+            {'count': 1, 'resource_type': 'project - ACTIVE'},
+            {'count': 1, 'resource_type': 'project - DELETE PENDING'}]
+
+        expected_summary_data_send_email = (
+            [
+                {'count': 2, 'resource_type': 'bucket'},
+                {'count': 4, 'resource_type': 'dataset'},
+                {'count': 1, 'resource_type': 'folder'},
+                {'count': 1, 'resource_type': 'object'},
+                {'count': 1, 'resource_type': 'organization'},
+                {'count': 2, 'resource_type': 'project'}],
+            [
+                {'count': 2, 'resource_type': 'dataset - HIDDEN'},
+                {'count': 2, 'resource_type': 'dataset - SHOWN'},
+                {'count': 1, 'resource_type': 'project - ACTIVE'},
+                {'count': 1, 'resource_type': 'project - DELETE PENDING'}])
 
         self.assertEquals(1, notifier._upload_to_gcs.call_count)
         self.assertEquals(
-            expected_summary_data,
+            expected_summary_data_upload_to_gcs,
             notifier._upload_to_gcs.call_args[0][0])
 
         self.assertEquals(1, notifier._send_email.call_count)
         self.assertEquals(
-            expected_summary_data,
-            notifier._send_email.call_args[0][0])
+            expected_summary_data_send_email,
+            notifier._send_email.call_args[0])
+
 
 if __name__ == '__main__':
     unittest.main()