Merge branch 'develop'

.circleci/config.yml

@@ -4,11 +4,7 @@
 #
 version: 2
 jobs:
-  build:
-    #branches:
-      #only:
-        #- feature/circleci
-
+  test:
     docker:
       # specify the version you desire here
       # use `-browsers` prefix for selenium tests, e.g. `3.6.1-browsers`
@@ -90,3 +86,34 @@ jobs:
           command: |
             . env/bin/activate
             flake8 OIPA
+  deploy:
+    docker:
+        - image: circleci/python:3.6.5
+    steps:
+        - checkout
+        - run:
+              name: Add target host to known_hosts files
+              command: |
+                  if [ "${CIRCLE_BRANCH}" == "develop" ]; then
+                    ssh-keyscan "$SSH_HOST_STAGING" >> ~/.ssh/known_hosts
+                  fi
+        - add_ssh_keys:
+              name: Add SSH KEY
+              fingerprints:
+                  - "5d:59:e5:e3:ce:cf:ec:18:62:3b:e6:fa:86:c7:9b:9a"
+        - run: eval ssh-agent -s
+        - run:
+              name: Deploy
+              command: |
+                  if [ "${CIRCLE_BRANCH}" == "develop" ]; then
+                    ssh "$SSH_USER"@"$SSH_HOST_STAGING" "cd OIPA/OIPA; source env/bin/activate; echo '"$SSH_STAGING_PASS"' | sudo -S git pull; pip install -r requirements.txt; ./manage.py migrate; echo '"$SSH_STAGING_PASS"' | sudo -S service supervisor restart"
+                  fi
+
+workflows:
+  version: 2
+  test_and_deploy:
+    jobs:
+      - test
+      - deploy:
+            requires:
+                - test

OIPA/OIPA/settings.py

@@ -212,7 +212,6 @@ def rel(*x):
         'rest_framework.parsers.JSONParser',
     ),
     'DEFAULT_RENDERER_CLASSES': (
-        'rest_framework.renderers.BrowsableAPIRenderer',
         'rest_framework.renderers.JSONRenderer',
         'api.renderers.PaginatedCSVRenderer',
         'api.renderers.XlsRenderer',
@@ -272,7 +271,7 @@ def rel(*x):
     'REGISTER_SERIALIZER': 'api.permissions.serializers.RegistrationSerializer'
 }
 
-EXPORT_COMMENT = 'Published with tools developed by Zimmerman & Zimmerman'
+# EXPORT_COMMENT = 'Published with tools developed by Zimmerman & Zimmerman'
 
 FIXTURE_DIRS = (
     os.path.join(BASE_DIR, '../fixtures/'),

OIPA/OIPA/urls.py

@@ -16,9 +16,7 @@
     url(r'^admin/doc/', include('django.contrib.admindocs.urls')),
     url(r'^admin/', admin.site.urls),
     url(r'^api/', include('api.urls')),
-    url(r'^home$',
-        TemplateView.as_view(template_name='home/home.html'),
-        name='home'),
+    url(r'^home$', view=views.home),
     url(r'^404$', views.error404),
     url(r'^500$', views.error500),
     url(r'^about$', TemplateView.as_view(template_name='home/about.html')),

OIPA/OIPA/views.py

@@ -1,5 +1,6 @@
 # Imports
 from django.http import HttpResponse
+from django.shortcuts import redirect
 from django.template import loader
 
 
@@ -32,3 +33,7 @@ def error500(request):
     # Return Template for this view + Data
     return HttpResponse(content=template.render(context),
                         content_type='text/html; charset=utf-8', status=500)
+
+
+def home(request):
+    return redirect('/api')

OIPA/api/activity/filters.py

@@ -45,6 +45,10 @@ class ActivityFilter(TogetherFilterSet):
         lookup_expr='in',
     )
 
+    is_secondary_reporter = BooleanFilter(
+        name='reporting_organisations__secondary_reporter',
+        widget=BooleanWidget())
+
     has_crs_add = IsNullBooleanFilter(name='crsadd', lookup_expr='isnull',
                                       distinct=True,
                                       widget=BooleanWidget())
@@ -394,7 +398,7 @@ class ActivityFilter(TogetherFilterSet):
     sector_vocabulary = ToManyFilter(
         qs=ActivitySector,
         lookup_expr='in',
-        name='sector__vocabulary__code',
+        name='vocabulary__code',
         fk='activity',
     )
 
@@ -452,6 +456,13 @@ class ActivityFilter(TogetherFilterSet):
         fk='activity',
     )
 
+    reporting_organisation_type = ToManyFilter(
+        qs=ActivityReportingOrganisation,
+        lookup_expr='in',
+        name='type',
+        fk='activity'
+    )
+
     reporting_organisation_identifier_startswith = ToManyFilter(
         qs=ActivityReportingOrganisation,
         lookup_expr='startswith',

OIPA/api/activity/serializers.py

@@ -2572,7 +2572,7 @@ class CrsAddSerializer(ModelSerializerNoValidation):
     other_flags = CrsAddOtherFlagsSerializer(many=True, required=False)
     loan_terms = CrsAddLoanTermsSerializer(required=False)
     loan_status = CrsAddLoanStatusSerializer(required=False)
-    channel_code = serializers.CharField(required=False)
+    channel_code = CodelistSerializer()
     activity = serializers.CharField(write_only=True)
 
     class Meta:
@@ -3286,6 +3286,11 @@ class ActivitySerializer(DynamicFieldsModelSerializer):
         read_only=True,
         view_name='activities:activity-transactions',
     )
+    transaction_url = serializers.HyperlinkedIdentityField(
+        read_only=True,
+        view_name='activities:activity-transactions-by-iati-identifier',
+        lookup_field='iati_identifier'
+    )
 
     related_transactions = TransactionSerializer(
         many=True,
@@ -3475,26 +3480,10 @@ def to_representation(self, instance):
         """
         Custom render to avoid auto render of related transaction
         """
-        request = self.context.get('request', None)
-        if request:
-            request_fields = request.GET.get('fields', None)
-        else:
-            request_fields = None
-
         ret = OrderedDict()
         fields = self._readable_fields
 
         for field in fields:
-            # Some activity has many related transaction,
-            # the request will take too long.
-            # So related transaction will be shown if it is on fields request
-            if field.field_name == 'related_transactions':
-                if request_fields:
-                    if 'related_transactions' not in request_fields.split(','):
-                        continue
-                else:
-                    continue
-
             try:
                 attribute = field.get_attribute(instance)
             except SkipField:
@@ -3512,7 +3501,61 @@ def to_representation(self, instance):
             if check_for_none is None:
                 ret[field.field_name] = None
             else:
-                ret[field.field_name] = field.to_representation(attribute)
+                # Some activity has many related transaction,
+                # the request will take too long.
+                # So related transaction will be shown if it is on fields
+                # request
+                if field.field_name in ['related_transactions',
+                                        'budgets',
+                                        'results',
+                                        ]:
+
+                    if field.field_name not in ret:
+                        if field.field_name == 'related_transactions' and \
+                                instance.transaction_set.count() > 100:
+                            custom_ret1 = OrderedDict()
+                            custom_ret1['message'] = \
+                                'This activity has more ' \
+                                'than 100 transactions! ' \
+                                'To get all transactions,  ' \
+                                'please use the transaction endpoint ' \
+                                'instead!'
+
+                            ret[field.field_name] = custom_ret1
+
+                        elif field.field_name == 'results' and \
+                                instance.result_set.count() > 100:
+                            custom_ret2 = OrderedDict()
+                            custom_ret2['message'] = \
+                                'This activity has more ' \
+                                'than 100 results! ' \
+                                'To get all results,  ' \
+                                'please use the results ' \
+                                'endpoint instead!'
+
+                            ret[field.field_name] = custom_ret2
+
+                        elif field.field_name == 'budgets' \
+                                and \
+                                instance.budget_set.count() > \
+                                100:
+                            custom_ret3 = OrderedDict()
+                            custom_ret3['message'] = \
+                                'This activity has more ' \
+                                'than 100 budgets! ' \
+                                'To get all budgets,  ' \
+                                'please use the budget ' \
+                                'endpoint instead!'
+
+                            ret[field.field_name] = custom_ret3
+
+                        else:
+                            ret[field.field_name] = field.to_representation(
+                                attribute
+                            )
+
+                else:
+                    ret[field.field_name] = field.to_representation(attribute)
 
         return ret
 
@@ -3569,7 +3612,8 @@ class Meta:
             'dataset',
             'publisher',
             'published_state',
-            'transaction_types'
+            'transaction_types',
+            'transaction_url'
         )
 
         validators = []
@@ -3581,3 +3625,9 @@ class ActivitySerializerByIatiIdentifier(ActivitySerializer):
         lookup_field='iati_identifier',
         read_only=True
     )
+
+
+class ActivityDetailSerializer(ActivitySerializer):
+    def to_representation(self, instance):
+        return super(ActivitySerializer, self).to_representation(
+            instance=instance)

OIPA/api/activity/tests/test_prefetches.py

@@ -472,7 +472,7 @@ def test_prefetch_crs_add(self):
         3. Fetch crsaddotherflags objects
         """
 
-        with self.assertNumQueries(3):
+        with self.assertNumQueries(6):
             queryset = Activity.objects.all().prefetch_crs_add()
             serializer = ActivitySerializer(
                 queryset,

OIPA/api/activity/tests/test_save_serializers.py

@@ -3994,7 +3994,7 @@ def test_create_crs_add(self):
                 "principal_arrears": 0,
                 "interest_arrears": 0,
             },
-            "channel_code": 21039,
+
         }
 
         res = self.c.post(
@@ -4078,7 +4078,7 @@ def test_update_crs_add(self):
                 "principal_arrears": 0,
                 "interest_arrears": 0,
             },
-            "channel_code": 21039,
+
         }
 
         res = self.c.put(

OIPA/api/activity/views.py

@@ -15,19 +15,19 @@
     ActivityAggregationFilter, ActivityFilter, RelatedOrderingFilter
 )
 from api.activity.serializers import (
-    ActivityDateSerializer, ActivityPolicyMarkerSerializer,
-    ActivityRecipientRegionSerializer, ActivitySectorSerializer,
-    ActivitySerializer, ActivitySerializerByIatiIdentifier,
-    BudgetItemSerializer, BudgetSerializer, CodelistSerializer,
-    ConditionSerializer, ConditionsSerializer, ContactInfoSerializer,
-    CountryBudgetItemsSerializer, CrsAddOtherFlagsSerializer, CrsAddSerializer,
-    DescriptionSerializer, DocumentLinkCategorySerializer,
-    DocumentLinkLanguageSerializer, DocumentLinkSerializer,
-    FssForecastSerializer, FssSerializer, HumanitarianScopeSerializer,
-    LegacyDataSerializer, LocationSerializer, OtherIdentifierSerializer,
-    ParticipatingOrganisationSerializer, PlannedDisbursementSerializer,
-    RecipientCountrySerializer, RelatedActivitySerializer,
-    ReportingOrganisationSerializer,
+    ActivityDateSerializer, ActivityDetailSerializer,
+    ActivityPolicyMarkerSerializer, ActivityRecipientRegionSerializer,
+    ActivitySectorSerializer, ActivitySerializer,
+    ActivitySerializerByIatiIdentifier, BudgetItemSerializer, BudgetSerializer,
+    CodelistSerializer, ConditionSerializer, ConditionsSerializer,
+    ContactInfoSerializer, CountryBudgetItemsSerializer,
+    CrsAddOtherFlagsSerializer, CrsAddSerializer, DescriptionSerializer,
+    DocumentLinkCategorySerializer, DocumentLinkLanguageSerializer,
+    DocumentLinkSerializer, FssForecastSerializer, FssSerializer,
+    HumanitarianScopeSerializer, LegacyDataSerializer, LocationSerializer,
+    OtherIdentifierSerializer, ParticipatingOrganisationSerializer,
+    PlannedDisbursementSerializer, RecipientCountrySerializer,
+    RelatedActivitySerializer, ReportingOrganisationSerializer,
     ResultIndicatorPeriodActualDimensionSerializer,
     ResultIndicatorPeriodActualLocationSerializer,
     ResultIndicatorPeriodSerializer,
@@ -44,7 +44,6 @@
     DynamicDetailCRUDView, DynamicDetailView, DynamicListCRUDView,
     DynamicListView, SaveAllSerializer
 )
-from api.organisation.serializers import OrganisationSerializer
 from api.publisher.permissions import PublisherPermissions
 from api.region.serializers import RegionSerializer
 from api.sector.serializers import SectorSerializer
@@ -59,9 +58,9 @@
     BudgetItem, CollaborationType, Condition, Conditions, ContactInfo,
     CountryBudgetItem, CrsAdd, CrsAddOtherFlags, Description, DocumentCategory,
     DocumentLink, DocumentLinkCategory, DocumentLinkLanguage, Fss, FssForecast,
-    HumanitarianScope, LegacyData, Location, Organisation, OrganisationType,
-    OtherIdentifier, PlannedDisbursement, PolicySignificance, RelatedActivity,
-    Result, ResultIndicator, ResultIndicatorPeriod,
+    HumanitarianScope, LegacyData, Location, OrganisationType, OtherIdentifier,
+    PlannedDisbursement, PolicySignificance, RelatedActivity, Result,
+    ResultIndicator, ResultIndicatorPeriod,
     ResultIndicatorPeriodActualDimension, ResultIndicatorPeriodTargetDimension,
     ResultIndicatorReference, Sector
 )
@@ -188,11 +187,10 @@ class ActivityAggregations(AggregationView):
         ),
         GroupBy(
             query_param="reporting_organisation",
-            fields="reporting_organisations__organisation__id",
-            renamed_fields="reporting_organisation",
-            queryset=Organisation.objects.all(),
-            serializer=OrganisationSerializer,
-            serializer_main_field='id',
+            fields=("reporting_organisations__organisation__primary_name",
+                    "reporting_organisations__organisation__organisation_identifier"),  # NOQA: E501
+            renamed_fields=("reporting_organisation",
+                            "reporting_organisation_identifier"),
             name_search_field="reporting_organisations__organisation__primary_name",  # NOQA: E501
             renamed_name_search_field="reporting_organisation_name"
         ),
@@ -500,7 +498,7 @@ class ActivityDetail(DynamicDetailView):
 
     # TODO: filter_class, selectable_fields, etc. Is needed for detail?
     filter_class = ActivityFilter
-    serializer_class = ActivitySerializer
+    serializer_class = ActivityDetailSerializer
     selectable_fields = ()
 
     # specification document

OIPA/api/dataset/serializers.py

@@ -113,7 +113,13 @@ def get_filetype(self, obj):
     def get_activities(self, obj):
         request = self.context.get('request')
         url = request.build_absolute_uri(reverse('activities:activity-list'))
-        return url + '?dataset=' + str(obj.id)
+        try:
+            request_format = self.context.get('request').query_params.get(
+                'format')
+        except AttributeError:
+            request_format = ''
+        return url + '?dataset=' + str(obj.id) + '&format={request_format}'.\
+            format(request_format=request_format)
 
     def get_activity_count(self, obj):
         return Activity.objects.filter(dataset=obj.id).count()

OIPA/api/iati/elements.py

@@ -100,8 +100,12 @@ def __init__(self, data, key, dict_key):
 
         # if type data is dict then use the dict key
         if dict_key and isinstance(data, dict):
-            d_data = data.get(dict_key)
-            value = self.convert_to_string(d_data.get(key))
+            try:
+                d_data = data.get(dict_key)
+                if d_data:
+                    value = self.convert_to_string(d_data.get(key))
+            except KeyError:
+                pass
         elif not dict_key and isinstance(data, dict):
             value = self.convert_to_string(data.get(key))
         elif not isinstance(data, list):