Merge pull request #1309 from glogiotatidis/1308-nightly-release

[Fix #1308] Include Release Jobs in Nightly Bundles.

snippets/base/management/commands/generate_bundles.py

@@ -59,11 +59,24 @@ def handle(self, *args, **options):
             distributions = distribution_bundle.distributions.all()
 
             for channel, locale in combinations_to_process:
+                additional_jobs = []
+                if channel == 'nightly' and settings.NIGHTLY_INCLUDES_RELEASE:
+                    additional_jobs = Job.objects.filter(
+                        status=Job.PUBLISHED).filter(**{
+                            'targets__on_release': True,
+                            'distribution__in': distributions,
+                        })
+
                 channel_jobs = Job.objects.filter(
-                    status=Job.PUBLISHED).filter(**{
-                        'targets__on_{}'.format(channel): True,
-                        'distribution__in': distributions,
-                    })
+                    status=Job.PUBLISHED).filter(
+                        Q(**{
+                            'targets__on_{}'.format(channel): True,
+                            'distribution__in': distributions,
+                        }))
+
+                all_jobs = Job.objects.filter(
+                    Q(id__in=additional_jobs) | Q(id__in=channel_jobs)
+                )
 
                 locales_to_process = [
                     key.lower() for key in product_details.languages.keys()
@@ -79,9 +92,9 @@ def handle(self, *args, **options):
                     filename = os.path.join(settings.MEDIA_BUNDLES_PREGEN_ROOT, filename)
                     full_locale = ',{},'.format(locale_to_process.lower())
                     splitted_locale = ',{},'.format(locale_to_process.lower().split('-', 1)[0])
-                    bundle_jobs = channel_jobs.filter(
+                    bundle_jobs = all_jobs.filter(
                         Q(snippet__locale__code__contains=splitted_locale) |
-                        Q(snippet__locale__code__contains=full_locale))
+                        Q(snippet__locale__code__contains=full_locale)).distinct()
 
                     # If there 're no Published Jobs for the channel / locale /
                     # distribution combination, delete the current bundle file if
@@ -92,12 +105,21 @@ def handle(self, *args, **options):
                             default_storage.delete(filename)
                         continue
 
-                    data = [job.render() for job in bundle_jobs]
+                    data = []
+                    channel_job_ids = list(channel_jobs.values_list('id', flat=True))
+                    for job in bundle_jobs:
+                        if job.id in channel_job_ids:
+                            render = job.render()
+                        else:
+                            render = job.render(always_eval_to_false=True)
+                        data.append(render)
+
                     bundle_content = json.dumps({
                         'messages': data,
                         'metadata': {
                             'generated_at': datetime.utcnow().isoformat(),
                             'number_of_snippets': len(data),
+                            'channel': channel,
                         }
                     })
 

snippets/base/models.py

@@ -1864,7 +1864,13 @@ def clean(self):
              self.publish_start >= self.publish_end)):
             raise ValidationError('Publish start must come before publish end.')
 
-    def render(self):
+    def render(self, always_eval_to_false=False):
+        """Argument always_eval_to_false supports bundle generation for the
+        nightly channel where we want all Release snippets to get
+        included for the Fx team to monitor JEXL engine performance,
+        but we don't want them to actually show. See #1308
+
+        """
         rendered_snippet = self.snippet.render()
 
         rendered_snippet['id'] = str(self.id)
@@ -1897,13 +1903,16 @@ def render(self):
             CHANNELS_MAP[channel] for channel in CHANNELS_MAP if channel in self.channels
         ])
         rendered_snippet = util.deep_search_and_replace(rendered_snippet, '[[channels]]', channels)
-
         # Add JEXL targeting
         rendered_snippet['targeting'] = ' && '.join(
             [target.jexl_expr for
              target in self.targets.all().order_by('id') if
              target.jexl_expr]
         )
+        if always_eval_to_false:
+            if rendered_snippet['targeting']:
+                rendered_snippet['targeting'] += ' && '
+            rendered_snippet['targeting'] += 'false'
 
         # Add Client Limits
         frequency = {}

snippets/base/tests/test_commands.py

@@ -461,6 +461,64 @@ def test_generation(self):
             set([str(published_job_1.id), str(published_job_2.id)])
         )
 
+    @override_settings(
+        MEDIA_BUNDLES_PREGEN_ROOT='pregen',
+        NIGHTLY_INCLUDES_RELEASE=True,
+    )
+    def test_nightly_includes_release(self):
+        release_job = JobFactory(
+            status=models.Job.PUBLISHED,
+            snippet__locale=',en,',
+            targets=[
+                TargetFactory(
+                    on_release=True, on_beta=True, on_nightly=False, on_esr=False, on_aurora=False)
+            ]
+        )
+        nightly_job = JobFactory(
+            status=models.Job.PUBLISHED,
+            snippet__locale=',en,',
+            targets=[
+                TargetFactory(
+                    on_release=True, on_beta=False, on_nightly=True, on_esr=False, on_aurora=False)
+            ]
+        )
+
+        # Beta only job, not to be included
+        JobFactory(
+            status=models.Job.PUBLISHED,
+            snippet__locale=',en,',
+            targets=[
+                TargetFactory(
+                    on_release=False, on_beta=True, on_nightly=False, on_esr=False, on_aurora=False)
+            ]
+        )
+
+        with patch.multiple('snippets.base.management.commands.generate_bundles',
+                            json=DEFAULT,
+                            product_details=DEFAULT,
+                            default_storage=DEFAULT) as mock:
+            mock['json'].dumps.return_value = ''
+            mock['product_details'].languages.keys.return_value = ['en-us']
+            call_command('generate_bundles', stdout=Mock())
+
+        # Loop to find the nighlty bundle
+        for arg_list in mock['json'].dumps.call_args_list:
+            if arg_list[0][0]['metadata']['channel'] == 'nightly':
+                self.assertEqual(
+                    len(arg_list[0][0]['messages']),
+                    2
+                )
+                self.assertEqual(
+                    set([arg_list[0][0]['messages'][0]['id'],
+                         arg_list[0][0]['messages'][1]['id']]),
+                    set([str(release_job.id), str(nightly_job.id)])
+                )
+                self.assertEqual(
+                    set([arg_list[0][0]['messages'][0]['targeting'],
+                         arg_list[0][0]['messages'][1]['targeting']]),
+                    set(['', 'false'])
+                )
+
     @override_settings(BUNDLE_BROTLI_COMPRESS=True)
     def test_brotli_called(self):
         with patch('snippets.base.management.commands.generate_bundles.brotli') as brotli_mock:

snippets/base/tests/test_models.py

@@ -762,6 +762,20 @@ def test_render(self):
 
         self.assertEqual(generated_output, expected_output)
 
+    def test_render_always_eval_to_false(self):
+        job = JobFactory.create(
+            weight=10, campaign__slug='demo-campaign',
+            targets=[
+                TargetFactory(on_release=False, on_beta=False,
+                              on_esr=False, on_aurora=False, on_nightly=True,
+                              jexl_expr='(la==lo)'),
+            ]
+        )
+        job.snippet.render = Mock()
+        job.snippet.render.return_value = {}
+        generated_output = job.render(always_eval_to_false=True)
+        self.assertEqual(generated_output['targeting'], '(la==lo) && false')
+
     def test_render_client_limits(self):
         # Combined
         job = JobFactory.create(

snippets/settings.py

@@ -389,3 +389,6 @@
 USE_PREGEN_BUNDLES = config('USE_PREGEN_BUNDLES', default=False, cast=bool)
 
 NOTEBOOK_ARGUMENTS = ['--ip', '0.0.0.0']
+
+
+NIGHTLY_INCLUDES_RELEASE = config('NIGHTLY_INCLUDES_RELEASE', default=False, cast=bool)