Merge pull request #882 from hdoupe/compute-maintenance

Merged #882

webapp/apps/btax/models.py

@@ -14,9 +14,10 @@
 from ..taxbrain.models import (SeparatedValuesField,
                                CommaSeparatedField)
 
+from ..taxbrain.behaviors import Hostnameable
 
 
-class BTaxSaveInputs(models.Model):
+class BTaxSaveInputs(Hostnameable, models.Model):
     """
     This model contains all the parameters for the tax model and the tax
     result.

webapp/apps/btax/views.py

@@ -41,7 +41,7 @@
 from ..taxbrain.helpers import (format_csv,
                                 is_wildcard)
 from ..taxbrain.views import denormalize, normalize
-from .compute import DropqComputeBtax, MockComputeBtax, JobFailError
+from .compute import DropqComputeBtax, MockComputeBtax, JobFailError, BTAX_WORKERS
 
 from ..constants import (METTR_TOOLTIP, METR_TOOLTIP, COC_TOOLTIP, DPRC_TOOLTIP,
                         START_YEAR)
@@ -403,7 +403,10 @@ def output_detail(request, pk):
         return render(request, 'btax/results.html', context)
 
     else:
-
+        if not model.check_hostnames(BTAX_WORKERS):
+            print('bad hostname', model.jobs_not_ready, BTAX_WORKERS)
+            return render_to_response('taxbrain/failed.html',
+                                      context={'is_btax': True})
         job_ids = model.job_ids
         jobs_to_check = model.jobs_not_ready
         if not jobs_to_check:

webapp/apps/dynamic/models.py

@@ -15,7 +15,8 @@
 
 from ..taxbrain.models import (CommaSeparatedField, SeparatedValuesField,
                                TaxSaveInputs, OutputUrl)
-from ..taxbrain.behaviors import Resultable, Fieldable, DataSourceable
+from ..taxbrain.behaviors import (Resultable, Fieldable, DataSourceable,
+                                  Hostnameable)
 from ..taxbrain import helpers as taxbrain_helpers
 from ..taxbrain import param_formatters
 
@@ -61,7 +62,7 @@ class Meta:
 
 
 class DynamicBehaviorSaveInputs(DataSourceable, Fieldable, Resultable,
-                                models.Model):
+                                Hostnameable, models.Model):
     """
     This model contains all the parameters for the dynamic behavioral tax
     model and the tax result.
@@ -147,7 +148,7 @@ class Meta:
         )
 
 
-class DynamicElasticitySaveInputs(DataSourceable, models.Model):
+class DynamicElasticitySaveInputs(DataSourceable, Hostnameable, models.Model):
     """
     This model contains all the parameters for the dynamic elasticity
     wrt GDP dynamic macro model and tax result

webapp/apps/dynamic/views.py

@@ -43,6 +43,7 @@
 from ..taxbrain.helpers import (taxcalc_results_to_tables,
                                 convert_val)
 from ..taxbrain.param_displayers import default_behavior
+from ..taxbrain.compute import JobFailError, DROPQ_WORKERS
 from .helpers import (default_parameters, job_submitted,
                       ogusa_results_to_tables, success_text,
                       failure_text, normalize, denormalize, strip_empty_lists,
@@ -650,7 +651,9 @@ def elastic_results(request, pk):
         return render(request, 'dynamic/elasticity_results.html', context)
 
     else:
-
+        if not model.check_hostnames(DROPQ_WORKERS):
+            print('bad hostname', model.jobs_not_ready, DROPQ_WORKERS)
+            raise render_to_response('taxbrain/failed.html')
         job_ids = model.job_ids
         jobs_to_check = model.jobs_not_ready
         if not jobs_to_check:
@@ -839,7 +842,9 @@ def behavior_results(request, pk):
         return render(request, 'taxbrain/results.html', context)
 
     else:
-
+        if not model.check_hostnames(DROPQ_WORKERS):
+            print('bad hostname', model.jobs_not_ready, DROPQ_WORKERS)
+            raise render_to_response('taxbrain/failed.html')
         job_ids = model.job_ids
         jobs_to_check = model.jobs_not_ready
         if not jobs_to_check:

webapp/apps/taxbrain/behaviors.py

@@ -132,3 +132,24 @@ def use_puf_not_cps(self):
             return True
         else:
             return False
+
+
+class Hostnameable(models.Model):
+    """
+    Mix-in for providing functionality around hostnames
+    """
+    class Meta:
+        abstract = True
+
+    def check_hostnames(self, current_hostnames):
+        """
+        Make sure all hostnames are valid before posting to/getting data from
+        them
+        """
+        if self.jobs_not_ready is None:
+            return True
+        for id in self.jobs_not_ready:
+            hn = id.split('#')[1]
+            if hn not in current_hostnames:
+                return False
+        return True

webapp/apps/taxbrain/compute.py

@@ -4,7 +4,7 @@
 import json
 import requests
 import taxcalc
-from requests.exceptions import Timeout, RequestException
+from requests.exceptions import Timeout, RequestException, ConnectionError
 from .helpers import arrange_totals_by_row
 from ..constants import START_YEAR
 import requests_mock
@@ -317,6 +317,16 @@ def remote_results_ready(self, theurl, params):
             mock.register_uri('GET', '/dropq_query_result', text='FAIL')
             return DropqCompute.remote_results_ready(self, theurl, params)
 
+class MockFailedComputeOnOldHost(MockCompute):
+    """
+    Simulate requesting results from a host IP that is no longer used. This
+    action should raise a `ConnectionError`
+    """
+    def remote_results_ready(self, theurl, params):
+        print 'MockFailedComputeOnOldHost remote_results_ready', theurl, params
+        raise requests.ConnectionError()
+
+
 class NodeDownCompute(MockCompute):
 
     __slots__ = ('count', 'num_times_to_wait', 'switch')

webapp/apps/taxbrain/models.py

@@ -17,7 +17,7 @@
 import helpers
 import param_formatters
 
-from behaviors import Resultable, Fieldable, DataSourceable
+from behaviors import Resultable, Fieldable, DataSourceable, Hostnameable
 
 
 # digit or true/false (case insensitive)
@@ -86,7 +86,8 @@ class ErrorMessageTaxCalculator(models.Model):
     text = models.CharField(blank=True, null=False, max_length=4000)
 
 
-class TaxSaveInputs(DataSourceable, Fieldable, Resultable, models.Model):
+class TaxSaveInputs(DataSourceable, Fieldable, Resultable, Hostnameable,
+                    models.Model):
     """
     This model contains all the parameters for the tax model and the tax
     result.

webapp/apps/taxbrain/tests/test_views.py

@@ -12,7 +12,7 @@
 from ..helpers import (expand_1D, expand_2D, expand_list, package_up_vars,
                      format_csv, arrange_totals_by_row, default_taxcalc_data)
 from ..compute import (DropqCompute, MockCompute, MockFailedCompute,
-                       NodeDownCompute)
+                       NodeDownCompute, MockFailedComputeOnOldHost)
 from ..views import get_result_context
 import taxcalc
 from taxcalc import Policy
@@ -815,3 +815,41 @@ def test_get_not_avail_page_renders(self, start_year, start_year_is_none):
                     for t in response.templates])
         edit_exp = '/taxbrain/edit/{}/?start_year={}'.format(pk, start_year)
         assert response.context['edit_href'] == edit_exp
+
+    def test_get_failed_sim_on_old_host(self, monkeypatch):
+        """
+        Simulate retrieving results from a host that is no longer used. Prior
+        to fix this raised a `requests.ConnectionError`
+        """
+        start_year = 2018
+        data = get_post_data(start_year)
+        data['first_year'] = start_year
+
+        result = do_micro_sim(CLIENT, data)
+        # get DB object
+        pk = result['pk']
+        url = OutputUrl.objects.filter(pk=pk)
+        assert len(url) == 1
+        model = url[0].unique_inputs
+        # swap out the job_ids to an IP address that is not
+        # used anymore,
+        model.job_ids = [
+            u'abc#1.1.1.1',
+            u'def#2.2.2.2',
+            u'ghi#3.3.3.3',
+            u'jkl#4.4.4.4',
+        ]
+        # simulate having some unfinished jobs and no result
+        model.jobs_not_ready = model.job_ids[:2]
+        model.tax_result = None
+        model.save()
+
+        from ...taxbrain import views
+        monkeypatch.setattr(views, 'dropq_compute', MockFailedComputeOnOldHost())
+
+        # try to get the bad results page
+        results_url = '/taxbrain/{}/'.format(pk)
+        response = CLIENT.get(results_url)
+
+        assert any([t.name == 'taxbrain/failed.html'
+                    for t in response.templates])

webapp/apps/taxbrain/views.py

@@ -7,6 +7,7 @@
 import tempfile
 import re
 import traceback
+import requests
 
 
 #Mock some module for imports because we can't fit them on Heroku slugs
@@ -44,7 +45,7 @@
                       )
 from .param_displayers import nested_form_parameters
 from .compute import (DropqCompute, MockCompute, JobFailError, NUM_BUDGET_YEARS,
-                      NUM_BUDGET_YEARS_QUICK)
+                      NUM_BUDGET_YEARS_QUICK, DROPQ_WORKERS)
 
 dropq_compute = DropqCompute()
 
@@ -780,7 +781,9 @@ def output_detail(request, pk):
     elif model.error_text:
         return render(request, 'taxbrain/failed.html', {"error_msg": model.error_text.text})
     else:
-
+        if not model.check_hostnames(DROPQ_WORKERS):
+            print('bad hostname', model.jobs_not_ready, DROPQ_WORKERS)
+            return render_to_response('taxbrain/failed.html')
         job_ids = model.job_ids
         jobs_to_check = model.jobs_not_ready
         if not jobs_to_check:

webapp/apps/test_assets/test_models.py

@@ -7,7 +7,10 @@
 from datetime import datetime
 
 from ..taxbrain.models import (JSONReformTaxCalculator,
-                               OutputUrl)
+                               OutputUrl, TaxSaveInputs)
+from ..dynamic.models import (DynamicBehaviorSaveInputs,
+                              DynamicElasticitySaveInputs)
+from ..btax.models import BTaxSaveInputs
 from utils import get_taxbrain_model, stringify_fields
 from ..taxbrain.forms import TaxBrainForm
 
@@ -76,3 +79,47 @@ def parse_fields(self, start_year, fields, Form=TaxBrainForm,
         # do some kind of check here using `exp_result`
 
         return model
+
+@pytest.mark.django_db
+class TestHostNames:
+
+    @pytest.mark.parametrize(
+        'Inputs',
+        [TaxSaveInputs, DynamicBehaviorSaveInputs, DynamicElasticitySaveInputs,
+         BTaxSaveInputs]
+    )
+    def test_check_hostnames(self, Inputs):
+        inputs = Inputs()
+        inputs.job_ids = []
+        inputs.job_ids = [
+            u'abc#1.1.1.1',
+            u'def#2.2.2.2',
+            u'ghi#3.3.3.3',
+            u'jkl#4.4.4.4',
+        ]
+        inputs.jobs_not_ready = inputs.job_ids[:2]
+        inputs.save()
+        assert inputs.check_hostnames(['1.1.1.1', '2.2.2.2', '3.3.3.3', '4.4.4.4'])
+
+
+    @pytest.mark.parametrize(
+        'Inputs',
+        [TaxSaveInputs, DynamicBehaviorSaveInputs, DynamicElasticitySaveInputs,
+         BTaxSaveInputs]
+    )
+    def test_check_hostnames_false(self, Inputs):
+        inputs = Inputs()
+        inputs.job_ids = [
+            u'abc#1.1.1.1',
+            u'def#2.2.2.2',
+            u'ghi#3.3.3.3',
+            u'jkl#4.4.4.4',
+        ]
+        inputs.jobs_not_ready = inputs.job_ids[:2]
+        inputs.save()
+        assert not inputs.check_hostnames(['5.5.5.5'])
+
+    def test_check_hostnames_none(self):
+        inputs = TaxSaveInputs()
+        inputs.jobs_not_ready = None
+        assert inputs.check_hostnames([])