OpenConceptLab/ocl_issues#412 | scripts to import v1 users and orgs

.coveragerc

@@ -6,6 +6,7 @@ omit =
     core/celery.py
     core/common/healthcheck/*
     core/wsgi.py
+    */v1_dump/*
     */management/commands/*
     */migrations/*
     */documents.py

.pylintrc

@@ -1,7 +1,7 @@
 [MESSAGES CONTROL]
 disable=missing-docstring,too-few-public-methods,no-member,too-many-ancestors,attribute-defined-outside-init,fixme,broad-except,import-outside-toplevel,too-many-instance-attributes,unsupported-membership-test,too-many-public-methods,too-many-lines
 [MASTER]
-ignore=migrations,fixtures,scripts,commands,documents.py,password_validation.py
+ignore=migrations,fixtures,scripts,commands,documents.py,password_validation.py,v1_dump
 [FORMAT]
 max-line-length=120
 [DESIGN]

core/importers/management/commands/import_v1_orgs.py

@@ -0,0 +1,66 @@
+import json
+
+from django.core.management import BaseCommand
+from pydash import get
+
+from core.common.tasks import populate_indexes
+from core.orgs.models import Organization
+from pprint import pprint
+
+
+class Command(BaseCommand):
+    help = 'import v1 orgs'
+
+    total = 0
+    processed = 0
+    created = []
+    existed = []
+    failed = []
+
+    @staticmethod
+    def log(msg):
+        print("*******{}*******".format(msg))
+
+    def handle(self, *args, **options):
+        FILE_PATH = '/code/core/importers/v1_dump/data/exported_orgs.json'
+        lines = open(FILE_PATH, 'r').readlines()
+
+        self.log('STARTING ORGS IMPORT')
+        self.total = len(lines)
+        self.log('TOTAL: {}'.format(self.total))
+
+        for line in lines:
+            data = json.loads(line)
+            original_data = data
+            self.processed += 1
+            _id = data.pop('_id')
+            created_at = data.pop('created_at')
+            updated_at = data.pop('updated_at')
+            data['internal_reference_id'] = get(_id, '$oid')
+            data['created_at'] = get(created_at, '$date')
+            data['updated_at'] = get(updated_at, '$date')
+            mnemonic = data.get('mnemonic')
+            self.log("Processing: {} ({}/{})".format(mnemonic, self.processed, self.total))
+            if Organization.objects.filter(mnemonic=mnemonic).exists():
+                self.existed.append(original_data)
+            else:
+                org = Organization.objects.create(**data)
+                if org:
+                    self.created.append(original_data)
+                else:
+                    self.failed.append(original_data)
+
+        populate_indexes.delay(['orgs'])
+
+        self.log(
+            "Result: Created: {} | Existed: {} | Failed: {}".format(
+                len(self.created), len(self.existed), len(self.failed)
+            )
+        )
+        if self.existed:
+            self.log("Existed")
+            pprint(self.existed)
+
+        if self.failed:
+            self.log("Failed")
+            pprint(self.failed)

core/importers/management/commands/import_v1_users.py

@@ -0,0 +1,92 @@
+import json
+from pprint import pprint
+
+from django.core.management import BaseCommand
+from pydash import get
+
+from core.common.tasks import populate_indexes
+from core.orgs.models import Organization
+from core.users.models import UserProfile
+
+
+class Command(BaseCommand):
+    help = 'import v1 users'
+
+    total = 0
+    processed = 0
+    created = []
+    existed = []
+    updated = []
+    failed = []
+
+    @staticmethod
+    def log(msg):
+        print("*******{}*******".format(msg))
+
+    def handle(self, *args, **options):
+        FILE_PATH = '/code/core/importers/v1_dump/data/exported_users.json'
+        lines = open(FILE_PATH, 'r').readlines()
+
+        self.log('STARTING USERS IMPORT')
+        self.total = len(lines)
+        self.log('TOTAL: {}'.format(self.total))
+
+        for line in lines:
+            data = json.loads(line)
+            original_data = data
+            self.processed += 1
+            data.pop('_id')
+            _id = data.pop('user_id')
+            # created_at = data.pop('created_at')
+            # updated_at = data.pop('updated_at')
+            last_login = data.pop('last_login')
+            date_joined = data.pop('date_joined')
+            full_name = data.pop('full_name') or ''
+            name_parts = list(set(full_name.split(' ')))
+            first_name = data.pop('first_name', '') or ' '.join(name_parts[:-1])
+            last_name = data.pop('last_name', '') or name_parts[-1]
+            orgs = data.pop('organizations', [])
+            password = data.pop('password')
+            hashed_password = data.pop('hashed_password')
+            password = password or hashed_password
+            data['verified'] = data.pop('verified_email', True)
+
+            data['last_name'] = last_name
+            data['internal_reference_id'] = get(_id, '$oid')
+            # data['created_at'] = get(created_at, '$date')
+            # data['updated_at'] = get(updated_at, '$date')
+            data['date_joined'] = get(date_joined, '$date')
+            data['last_login'] = get(last_login, '$date')
+            data['first_name'] = first_name
+            username = data.get('username')
+            self.log("Processing: {} ({}/{})".format(username, self.processed, self.total))
+            queryset = UserProfile.objects.filter(username=username)
+            if queryset.exists():
+                self.updated.append(original_data)
+                user = queryset.first()
+                user.organizations.set(Organization.objects.filter(internal_reference_id__in=orgs))
+            else:
+                try:
+                    user = UserProfile.objects.create(**data)
+                    if user:
+                        self.created.append(original_data)
+                        user.password = password
+                        user.save()
+                        user.organizations.set(Organization.objects.filter(internal_reference_id__in=orgs))
+                    else:
+                        self.failed.append(original_data)
+                except Exception as ex:
+                    self.failed.append({**original_data, 'errors': ex.args})
+        populate_indexes.delay(['users', 'orgs'])
+        self.log(
+            "Result: Created: {} | Updated: {} | Failed: {}".format(
+                len(self.created), len(self.updated), len(self.failed),
+            )
+        )
+        if self.existed:
+            self.log("Updated")
+            pprint(self.updated)
+
+        if self.failed:
+            self.log("Failed")
+            pprint(self.failed)

core/importers/v1_dump/scripts/export_orgs.bash

@@ -0,0 +1,3 @@
+#!/usr/bin/env bash
+
+mongoexport --db ocl --collection orgs_organization --fields=_id,mnemonic,website,created_at,updated_at,is_active,uri,extras,location,public_access,company,name -o ../data/exported_orgs.json

core/importers/v1_dump/scripts/export_users.bash

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+
+mongo "localhost:27017/ocl" ./export_user.js
+mongoexport --db ocl --collection export.results -o ../data/exported_users.json

core/importers/v1_dump/scripts/export_users.js

@@ -0,0 +1,14 @@
+db.export.results.drop();
+
+cursor = db.auth_user.find();
+
+while (cursor.hasNext()) {
+    auth_user = cursor.next();
+    profiles = db.users_userprofile.find({"user_id": auth_user._id}, {company: 1, location: 1, _id: 0, verified_email: 1, user_id: 1, hashed_password: 1, full_name: 1, preferred_locale: 1, organizations: 1, extras: 1, public_access: 1, verified_email: 1});
+    if (profiles.hasNext()) {
+        user_profile = profiles.next();
+        db.export.results.insert(Object.assign({}, auth_user, user_profile));
+    }
+}
+
+print(db.export.results.count() + " matching documents found");

core/users/migrations/0012_auto_20210304_0413.py

@@ -0,0 +1,23 @@
+# Generated by Django 3.0.9 on 2021-03-04 04:13
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('users', '0011_userprofile_verification_token'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='userprofile',
+            name='first_name',
+            field=models.CharField(blank=True, max_length=100),
+        ),
+        migrations.AlterField(
+            model_name='userprofile',
+            name='last_name',
+            field=models.CharField(blank=True, max_length=100),
+        ),
+    ]

core/users/models.py

@@ -19,6 +19,8 @@ class Meta:
         swappable = 'AUTH_USER_MODEL'
 
     OBJECT_TYPE = USER_OBJECT_TYPE
+    first_name = models.CharField(max_length=100, blank=True)
+    last_name = models.CharField(max_length=100, blank=True)
     organizations = models.ManyToManyField('orgs.Organization', related_name='members')
     company = models.TextField(null=True, blank=True)
     location = models.TextField(null=True, blank=True)

core/users/serializers.py

@@ -124,7 +124,7 @@ class Meta:
             'public_collections', 'public_sources', 'created_on', 'updated_on', 'created_by', 'updated_by',
             'url', 'organizations_url', 'extras', 'sources_url', 'collections_url', 'website', 'last_login',
             'logo_url', 'subscribed_orgs', 'is_superuser', 'is_staff', 'first_name', 'last_name', 'verified',
-            'verification_token',
+            'verification_token', 'date_joined',
         )
 
     def __init__(self, *args, **kwargs):