GP2-1398_comtrade_data_copy

CHANGELOG.md

@@ -1,6 +1,7 @@
 # Changelog
 
 ### Implemented enhancements
+- GP2-1398 - Comtrade data in database
 - GP2-1343 - Added Trading blocs data
 - GP2-1348 - Economy and population raw values and rank totals
 - GP2-1264 - Added currencies data
@@ -22,7 +23,7 @@
 - GP2-1382 - getting paid structure
 - GP2-1180 - travel bus BE
 - NOTICKET - Exportplan make dict default for JSON fields.
-- GP2-1181 - business risk
+- GP2-1181 - 1398business risk
 
 ## Pre release
 ### Bugs fixed

conf/settings.py

@@ -505,3 +505,5 @@
 EXPORTING_OPPORTUNITIES_API_BASIC_AUTH_PASSWORD = env.str('EXPORTING_OPPORTUNITIES_API_BASIC_AUTH_PASSWORD', '')
 EXPORTING_OPPORTUNITIES_API_BASE_URL = env.str('EXPORTING_OPPORTUNITIES_API_BASE_URL')
 EXPORTING_OPPORTUNITIES_API_SECRET = env.str('EXPORTING_OPPORTUNITIES_API_SECRET')
+
+COMTRADE_DATA_FILE_NAME = env.str('COMTRADE_DATA_FILE_NAME', default='comtrade-import-data.csv')

conf/urls.py

@@ -289,6 +289,11 @@
         dataservices.views.RetrieveCountryDataView.as_view(),
         name='dataservices-country-data',
     ),
+    url(
+        r'^dataservices/country-data/$',
+        dataservices.views.RetrieveDataByCountryView.as_view(),
+        name='dataservices-country-data-by-country',
+    ),
     url(
         r'^dataservices/lastyearimportdata/$',
         dataservices.views.RetrieveLastYearImportDataView.as_view(),
@@ -299,6 +304,11 @@
         dataservices.views.RetrieveLastYearImportDataFromUKView.as_view(),
         name='last-year-import-data-from-uk',
     ),
+    url(
+        r'^dataservices/lastyearimportdatabycountry/$',
+        dataservices.views.RetrieveLastYearImportDataByCountryView.as_view(),
+        name='last-year-import-data-by-country',
+    ),
     url(
         r'^dataservices/cia-factbook-data/$',
         dataservices.views.RetrieveCiaFactbooklDataView.as_view(),

dataservices/helpers.py

@@ -343,6 +343,15 @@ def get_historical_import_data(country, commodity_code):
     return historical_data
 
 
+def get_comtrade_data_by_country(commodity_code, country_list):
+    data = {}
+    for record in models.ComtradeReport.objects.filter(country__iso2__in=country_list, commodity_code=commodity_code):
+        iso_code = record.country.iso2
+        data[iso_code] = data.get(iso_code, [])
+        data[iso_code].append(serializers.ComTradeReportSerializer(record).data)
+    return data
+
+
 @TTLCache()
 def get_world_economic_outlook_data(country_code):
     data = []
@@ -408,6 +417,16 @@ def get_society_data(country):
     return society_data
 
 
+def deep_extend(o1, o2):
+    # Deep extend dict o2 onto o1.  o1 is mutated
+    for key, value in o2.items():
+        if o1.get(key) and isinstance(o1.get(key), dict) and isinstance(value, dict):
+            deep_extend(o1.get(key), value)
+        else:
+            o1[key] = value
+    return o1
+
+
 def millify(n):
     n = float(n)
     mill_names = ['', ' thousand', ' million', ' billion', ' trillion']
@@ -441,6 +460,13 @@ def get_serialized_instance_from_model(model_class, serializer_class, filter_arg
         return None
 
 
+def get_multiple_serialized_instance_from_model(model_class, serializer_class, filter_args, section_key):
+    out = {}
+    for result in model_class.objects.filter(**filter_args):
+        out[result.country.iso2] = {section_key: serializer_class(result).data}
+    return out
+
+
 def calculate_total_internet_population(internet_usage, total_population):
     if internet_usage and total_population:
         percent = float(internet_usage.get('value')) / 100

dataservices/management/commands/import_comtrade_data.py

@@ -0,0 +1,121 @@
+import csv
+
+from django.conf import settings
+from django.core.management import BaseCommand
+from django.db import connection
+
+from core.helpers import get_s3_file_stream
+from dataservices.models import ComtradeReport
+
+
+class Command(BaseCommand):
+    help = 'Import Comtrade data'
+
+    def add_arguments(self, parser):
+        # Positional arguments
+        parser.add_argument('filenames', nargs='*', type=str)
+
+        parser.add_argument(
+            '--wipe',
+            action='store_true',
+            help='Wipe table before loading',
+        )
+        parser.add_argument(
+            '--raw',
+            action='store_true',
+            help='load raw data files',
+        )
+
+        parser.add_argument(
+            '--test',
+            action='store_true',
+            help='limit rowcount to 1000 for testing',
+        )
+
+    def load_raw_files(self, filenames):
+        # Loads a raw file as downloaded from comtrade on top of existing data in db
+
+        for filename in filenames:
+            self.stdout.write(self.style.SUCCESS(f'********  Loading: {filename}'))
+            with open(filename, 'r', encoding='utf-8-sig') as f:
+                written = 0
+                read = 0
+                file_reader = csv.DictReader(f)
+                for row in file_reader:
+                    read = read + 1
+                    if row.get('Is Leaf Code') == '1':
+                        reporter_iso3 = row.get('Reporter ISO')
+                        partner_iso3 = row.get('Partner ISO')
+                        flow = row.get('Trade Flow')
+                        uk_or_world = None
+                        country_iso3 = None
+                        if reporter_iso3 == 'GBR' and flow == 'Export':
+                            uk_or_world = reporter_iso3
+                            country_iso3 = partner_iso3
+                        if partner_iso3 == 'WLD' and flow == 'Import':
+                            uk_or_world = partner_iso3
+                            country_iso3 = reporter_iso3
+                        if country_iso3 and uk_or_world:
+                            written = written + 1
+                            report = ComtradeReport(
+                                country_iso3=country_iso3,
+                                year=row.get('Year'),
+                                classification=row.get('Classification'),
+                                commodity_code=row.get('Commodity Code'),
+                                trade_value=float(row.get('Trade Value (US$)') or '0'),
+                                uk_or_world=uk_or_world,
+                            )
+                            report.save()
+                            if written % 100 == 0:
+                                print(f'{read} read, {written} written', end='\r', flush=True)
+                self.stdout.write(self.style.SUCCESS(f'{read} read, {written} written'))
+
+    def populate_db_from_s3(self, filename, test):
+        # Read from S3, write into local DB, hook up country table
+        cursor = connection.cursor()
+        filestream = get_s3_file_stream(filename or settings.COMTRADE_DATA_FILE_NAME)
+        file_reader = csv.DictReader(filestream.split())
+        self.stdout.write('*********   Loading comtrade data')
+        written = 0
+        for row in file_reader:
+            cursor.execute(
+                "INSERT INTO \
+                dataservices_comtradereport \
+                (id, year, classification, commodity_code, trade_value, uk_or_world, country_iso3 )\
+                VALUES\
+                (%s, %s, %s, %s, %s, %s, %s)",
+                [
+                    row.get('id'),
+                    row.get('year'),
+                    row.get('classification'),
+                    row.get('commodity_code'),
+                    row.get('trade_value'),
+                    row.get('uk_or_world'),
+                    row.get('country_iso3'),
+                ],
+            )
+
+            written = written + 1
+            if written % 1000 == 0:
+                print(f'  {written} rows written', end='\r', flush=True)
+            if written >= 1000 and test:
+                break
+        self.stdout.write(self.style.SUCCESS(f'Loaded table - {written} rows written'))
+
+        self.stdout.write('Linking countries')
+        cursor.execute(
+            "UPDATE dataservices_comtradereport as d \
+            set country_id=c.id \
+            from dataservices_country as c where d.country_iso3=c.iso3;"
+        )
+
+    def handle(self, *args, **options):
+        filenames = options['filenames']
+        if options['wipe']:
+            ComtradeReport.objects.all().delete()
+        elif filenames and options['raw']:
+            self.load_raw_files(filenames)
+        else:
+            self.populate_db_from_s3(filenames and filenames[0], test=options['test'])
+
+        self.stdout.write(self.style.SUCCESS('All done, bye!'))

dataservices/management/commands/tests/test_import_data.py

@@ -65,3 +65,32 @@ def test_import_rank_of_law_data_with_no_country():
     currencies_data = models.Currency.objects.first()
     assert rule_of_law.country is None
     assert currencies_data.country is None
+
+
+@pytest.mark.django_db
+def test_import_comtrade():
+    management.call_command('import_countries')
+    management.call_command('import_comtrade_data', '--test')
+    data = models.ComtradeReport.objects.filter(country_iso3='FRA', year='2019', commodity_code='010649')
+    assert len(models.ComtradeReport.objects.all()) == 1000
+    assert data.first().country.iso3 == 'FRA'
+    assert data.first().commodity_code == '010649'
+    assert len(data) == 2
+    assert data.first().trade_value == 9189567
+
+
+@pytest.mark.django_db
+def test_import_comtrade_raw():
+    management.call_command('import_comtrade_data', '--raw', 'dataservices/resources/comtrade_sample.csv')
+    data = models.ComtradeReport.objects.filter(country_iso3='FRA', commodity_code='390720')
+    assert len(models.ComtradeReport.objects.all()) == 389
+    assert len(data) == 2
+    assert str(data.first()) == 'FRA:390720'
+    assert data.first().country_iso3 == 'FRA'
+    assert data.first().commodity_code == '390720'
+    assert data.first().trade_value == 345434516
+    assert data.first().uk_or_world == 'WLD'
+    assert data[1].uk_or_world == 'GBR'
+
+    management.call_command('import_comtrade_data', '--wipe')
+    assert len(models.ComtradeReport.objects.all()) == 0

dataservices/migrations/0024_auto_20210219_0848.py

@@ -0,0 +1,31 @@
+# Generated by Django 2.2.13 on 2021-02-19 08:48
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('dataservices', '0023_auto_20210204_1009'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='ComtradeReport',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('year', models.IntegerField(blank=True, null=True)),
+                ('classification', models.CharField(max_length=3)),
+                ('country_iso3', models.CharField(max_length=3)),
+                ('uk_or_world', models.CharField(max_length=3)),
+                ('commodity_code', models.CharField(max_length=6)),
+                ('trade_value', models.DecimalField(blank=True, decimal_places=0, max_digits=15, null=True)),
+                ('country', models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, to='dataservices.Country', verbose_name='Countries')),
+            ],
+        ),
+        migrations.AddIndex(
+            model_name='comtradereport',
+            index=models.Index(fields=['commodity_code', 'country', 'uk_or_world'], name='dataservice_commodi_c61be8_idx'),
+        ),
+    ]

dataservices/models.py

@@ -197,3 +197,23 @@ class TradingBlocs(TimeStampedModel):
 
     class Meta:
         verbose_name = "Trading Bloc"
+
+
+class ComtradeReport(models.Model):
+    year = models.IntegerField(null=True, blank=True)
+    classification = models.CharField(unique=False, blank=False, null=False, max_length=3)
+    country_iso3 = models.CharField(unique=False, blank=False, null=False, max_length=3)
+    uk_or_world = models.CharField(unique=False, blank=False, null=False, max_length=3)
+    commodity_code = models.CharField(unique=False, blank=False, null=False, max_length=6)
+    trade_value = models.DecimalField(null=True, blank=True, decimal_places=0, max_digits=15)
+    country = models.ForeignKey(
+        'dataservices.Country', verbose_name=_('Countries'), on_delete=models.SET_NULL, null=True
+    )
+
+    class Meta:
+        indexes = [
+            models.Index(fields=['commodity_code', 'country', 'uk_or_world']),
+        ]
+
+    def __str__(self):
+        return str(self.country_iso3) + ':' + str(self.commodity_code)