Deprecate MySQL support #191

docs/conf.py

@@ -56,9 +56,9 @@
 # built documents.
 #
 # The short X.Y version.
-version = '1.5'
+version = '1.x'
 # The full version, including alpha/beta/rc tags.
-release = '1.5.0'
+release = '1.x'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.

docs/faq.rst

@@ -189,6 +189,8 @@ Only want to restore the database?
 
 - Stop the application first with ``sudo supervisorctl stop all``. This will disconnect it from the database as well.
 
+- Importing the data could take a long time. It took MySQL 15 minutes to import nearly 3 million readings, from a compressed backup, on a RaspberryPi 3. 
+
 For **PostgreSQL** restores::
 
     sudo sudo -u postgres dropdb dsmrreader

docs/installation.rst

@@ -121,7 +121,7 @@ Install MariaDB. You can also choose to install the closed source MySQL, as they
 
 - Create database::
 
-    sudo mysqladmin create dsmrreader
+    sudo mysqladmin --defaults-file=/etc/mysql/debian.cnf create dsmrreader
 
 - Create database user::
 

docs/locale/nl/LC_MESSAGES/faq.po

@@ -8,8 +8,8 @@ msgid ""
 msgstr ""
 "Project-Id-Version: DSMR Reader v1.x\n"
 "Report-Msgid-Bugs-To: Dennis Siemensma <github@dennissiemensma.nl>\n"
-"POT-Creation-Date: 2017-01-07 17:54+0100\n"
-"PO-Revision-Date: 2017-01-07 17:56+0100\n"
+"POT-Creation-Date: 2017-01-08 00:13+0100\n"
+"PO-Revision-Date: 2017-01-08 00:14+0100\n"
 "Last-Translator: Dennis Siemensma <github@dennissiemensma.nl>\n"
 "Language-Team: Dennis Siemensma <github@dennissiemensma.nl>\n"
 "MIME-Version: 1.0\n"
@@ -459,18 +459,27 @@ msgstr ""
 "ook voor dat de databaseverbinding van de applicatie verdwijnt."
 
 #: ../../faq.rst:192
+msgid ""
+"Importing the data could take a long time. It took MySQL 15 minutes to import "
+"nearly 3 million readings, from a compressed backup, on a RaspberryPi 3."
+msgstr ""
+"Het importeren van de gegevens kan enige tijd duren. MySQL doet er ongeveer 15 "
+"minuten over om bijna 3 miljoen metingen in te lezen op een RaspberryPi 3, "
+"vanuit een gecomprimeerde back-up."
+
+#: ../../faq.rst:194
 msgid "For **PostgreSQL** restores::"
 msgstr "Voor herstellen van **PostgreSQL**::"
 
-#: ../../faq.rst:206
+#: ../../faq.rst:208
 msgid "For **MySQL** restores::"
 msgstr "Voor herstellen van **MySQL**::"
 
-#: ../../faq.rst:220
+#: ../../faq.rst:222
 msgid "Start the application again with ``sudo supervisorctl start all``."
 msgstr "Start de applicatie weer met ``sudo supervisorctl start all``."
 
-#: ../../faq.rst:224
+#: ../../faq.rst:226
 msgid ""
 "In case the version differs, you can try forcing a deployment reload by: ``sudo "
 "su - dsmr`` and then executing ``./post-deploy.sh``."
@@ -479,27 +488,27 @@ msgstr ""
 "herladen. Log in met: ``sudo su - dsmr`` en voer vervolgens ``./post-deploy."
 "sh`` uit."
 
-#: ../../faq.rst:228
+#: ../../faq.rst:230
 msgid "How can I migrate from MySQL to PostgreSQL?"
 msgstr "Hoe kan ik migreren van MySQL naar PostgreSQL?"
 
-#: ../../faq.rst:232
+#: ../../faq.rst:234
 msgid ""
 "This is only relevant for users still running on MySQL since DSMR-reader "
 "``v1.5``."
 msgstr ""
 "Dit is alleen van toepassing op gebruikers van DSMR-reader vanaf ``v1.5`` in "
 "combinatie met MySQL."
 
-#: ../../faq.rst:234
+#: ../../faq.rst:236
 msgid ""
 "Start by :doc:`installing PostgreSQL as documented in chapter 1 in the "
 "installation guide<installation>`."
 msgstr ""
 "Start met :doc:`PostgreSQL te installeren, zoals beschreven in hoofdstuk 1 van "
 "de installatiehandleiding.<installation>`."
 
-#: ../../faq.rst:236
+#: ../../faq.rst:238
 msgid ""
 "Make sure you have enough diskspace before migrating. The following command "
 "will display the size of the MySQL database in MB. Execute it as ``root`` / "
@@ -509,19 +518,19 @@ msgstr ""
 "volgende command toont de grootte van de MySQL-database, in MB. Voer dit uit "
 "als ``root`` / ``sudo`` gebruiker::"
 
-#: ../../faq.rst:242
+#: ../../faq.rst:244
 msgid "View the available disk space on your system::"
 msgstr "Bekijk hoeveel schijfruimte er beschikbaar is::"
 
-#: ../../faq.rst:247
+#: ../../faq.rst:249
 msgid ""
 "This will display something similar as below and you'd (most likely) need to "
 "check the mount on ``/`` (indicated by ``<<<<<<<<<<`` in the example)::"
 msgstr ""
 "Dit toont een soortgelijke output als hieronder. Je zult waarschijnlijk moeten "
 "kijken naar de mount op ``/`` (in het voorbeeld gemarkeerd met ``<<<<<<<<<<``)::"
 
-#: ../../faq.rst:263
+#: ../../faq.rst:265
 msgid ""
 "In the example above the mount on ``/`` has ``11GB`` available. Do not continue "
 "when you're not sure whether you have enough disk space available."
@@ -530,20 +539,20 @@ msgstr ""
 "beschikbaar. Ga niet verder wanneer je niet zeker weet of je voldoende vrije "
 "schijfruimte beschikbaar hebt."
 
-#: ../../faq.rst:266
+#: ../../faq.rst:268
 msgid "Login as ``dsmr`` user::"
 msgstr "Login als ``dsmr`` gebruiker::"
 
-#: ../../faq.rst:271
+#: ../../faq.rst:273
 msgid "Now install the PostgreSQL connection client::"
 msgstr "Installeer nu de PostgreSQL client::"
 
-#: ../../faq.rst:276
+#: ../../faq.rst:278
 msgid "Initialize the (empty) PostgreSQL database you've just created::"
 msgstr ""
 "Initializeer de (lege) PostgreSQL-database die je zojuist hebt aangemaakt::"
 
-#: ../../faq.rst:285
+#: ../../faq.rst:287
 msgid ""
 "Now comes the waiting part. Please note that this **could take quite some "
 "time**, depending on how much data you have and the hardware used."
@@ -552,7 +561,7 @@ msgstr ""
 "beslag nemen**, afhankelijk van hoeveel gegevens er zijn en op welke hardware "
 "dit draait."
 
-#: ../../faq.rst:287
+#: ../../faq.rst:289
 msgid ""
 "However, you can run the migration simultaneously and multiple times, while "
 "keeping your source database collecting data. The migration command (the second "
@@ -562,7 +571,7 @@ msgstr ""
 "applicatie laten draaien. Het migratie-command (de tweede hieronder) gaat "
 "namelijk verder waar die is gebleven."
 
-#: ../../faq.rst:292
+#: ../../faq.rst:294
 msgid ""
 "After it's first completion, you should run the "
 "``dsmr_migrate_mysql_data_to_postgresql`` command one more time, to copy any "
@@ -572,23 +581,23 @@ msgstr ""
 "aan te raden deze nogmaals uit te voeren. Hiermee kopieer je recente gegevens "
 "mee over."
 
-#: ../../faq.rst:294
+#: ../../faq.rst:296
 msgid "Migrate data::"
 msgstr "Migreer gegevens::"
 
-#: ../../faq.rst:303
+#: ../../faq.rst:305
 msgid "When completed, it's time to switch the application config to PostgreSQL::"
 msgstr "Wanneer dit klaar is, kun je de applicatie wisselen naar PostgreSQL::"
 
-#: ../../faq.rst:308
+#: ../../faq.rst:310
 msgid "And now reload the application to apply changes::"
 msgstr "En herlaad de applicatie om de wijzigingen door te voeren::"
 
-#: ../../faq.rst:313
+#: ../../faq.rst:315
 msgid "You should now be running on PostgreSQL."
 msgstr "Je bent nu over naar PostgreSQL."
 
-#: ../../faq.rst:315
+#: ../../faq.rst:317
 msgid ""
 "Please check the Status page whether data is being processed properly. The page "
 "also displays whether you are now running on PostgreSQL (indicated after the "
@@ -598,63 +607,58 @@ msgstr ""
 "geeft ook aan of je nu op PostgreSQL draait (staat achter het versienummer op "
 "de pagina)."
 
-#: ../../faq.rst:316
+#: ../../faq.rst:318
 msgid ""
 "You should create or keep a **final backup of the MySQL database** before "
 "shutting it down and removing it's data."
 msgstr ""
 "Het is aan te raden om een **laatste back-up van de MySQL-database** te maken "
 "voordat je deze uitzet en compleet verwijdert."
 
-#: ../../faq.rst:318
+#: ../../faq.rst:320
 msgid ""
 "Due to technical reasons, any users created for the admin interface are **not** "
 "migrated. You can easily create a new user with::"
 msgstr ""
 "Omwille technische redenen, worden gebruikers van de admin-interface **niet** "
 "gemigreerd. Je kunt ze echter weer vrij eenvoudig aanmaken met::"
 
-#: ../../faq.rst:325
+#: ../../faq.rst:327
 msgid ""
 "In case anything went wrong, you can still easily go back to MySQL by restoring "
 "the config and reloading the application::"
 msgstr ""
 "Bij grote tegenvallers kun je altijd gemakkelijk terugwisselen naar MySQL. Zet "
 "dan de config terug en herlaad de applicatie::"
 
-#: ../../faq.rst:334
+#: ../../faq.rst:336
 msgid ""
 "**The next step below is irreversible if you do not have a backup of the MySQL "
 "data!**"
 msgstr ""
 "**De volgende stap is onomkeerbaar wanneer je geen back-up hebt van de MySQL-"
 "data!**"
 
-#: ../../faq.rst:336
+#: ../../faq.rst:338
 msgid ""
 "If you'd like to wipe MySQL from the system, go back to ``sudo`` / ``root`` "
 "user and execute::"
 msgstr ""
 "Wanneer je MySQL (inclusief data) helemaal wilt verwijderen, voer dan het "
 "volgende uit als ``sudo`` / ``root`` gebruiker::"
 
-#: ../../faq.rst:344
+#: ../../faq.rst:346
 msgid "Feature/bug report"
 msgstr "Verzoek/fout melden"
 
-#: ../../faq.rst:345
+#: ../../faq.rst:347
 msgid "*How can I propose a feature or report a bug I've found?*"
 msgstr "*Hoe kan ik een verzoek indienen of een fout melden?*"
 
-#: ../../faq.rst:349
+#: ../../faq.rst:351
 msgid ""
 "`Just create a ticket at Github <https://github.com/dennissiemensma/dsmr-reader/"
 "issues/new>`_."
 msgstr ""
 "`Maak een ticket aan op Github <https://github.com/dennissiemensma/dsmr-reader/"
 "issues/new>`_."
-
-#~ msgid "Please check the Status page whether data is being processed properly."
-#~ msgstr ""
-#~ "Controleer op de Status-pagina of de gegevens nog steeds goed verwerkt "
-#~ "worden."

dsmr_backend/management/commands/dsmr_migrate_mysql_data_to_postgresql.py

@@ -1,7 +1,7 @@
 from django.core.management.base import BaseCommand, CommandError
 from django.utils.translation import ugettext as _
 from django.db.utils import OperationalError, ConnectionDoesNotExist
-from django.db import connections, transaction
+from django.db import connections
 
 from dsmr_consumption.models.consumption import ElectricityConsumption, GasConsumption
 from dsmr_backup.models.settings import DropboxSettings, BackupSettings
@@ -24,8 +24,8 @@ class Command(BaseCommand):
     help = _('Migrates the data inside a MySQL database of this project to a PostgreSQL database.')
 
     # As defined in migration settings.
-    MYSQL_DB_KEY = 'default'
-    POSTGRESQL_DB_KEY = 'target'
+    MYSQL_DB_KEY = 'source'
+    POSTGRESQL_DB_KEY = 'default'
 
     OVERRIDE_MODELS = (
         APISettings, BackupSettings, DropboxSettings, ConsumptionSettings, DataloggerSettings, FrontendSettings,
@@ -86,6 +86,7 @@ def _migrate_resume(self):
         for current_model in self.APPEND_MODELS:
             source_dataset = current_model.objects.using(self.MYSQL_DB_KEY).all().order_by('pk')
 
+            print()
             print(' - {}'.format(current_model.__name__))
 
             try:
@@ -98,20 +99,17 @@ def _migrate_resume(self):
                 source_dataset = source_dataset.filter(pk__gt=latest_at_target.pk)
 
             # Chunk to indicate progress and keep memory usage low.
-            CHUNK_SIZE = 5000
+            CHUNK_SIZE = 1000
             total_count = source_dataset.count()
             print(' --- Total items to copy: {}'.format(total_count))
 
-            with transaction.atomic(using=self.POSTGRESQL_DB_KEY):
-                for start in range(0, total_count, CHUNK_SIZE):
-                    source_dataset_chunk = source_dataset[start:start + CHUNK_SIZE]
-                    print(' --- Processing {} item(s), {} remaining...'.format(
-                        source_dataset_chunk.count(), total_count
-                    ))
+            for start in range(0, total_count, CHUNK_SIZE):
+                source_dataset_chunk = source_dataset[start:start + CHUNK_SIZE]
+                print(' --- Processing {} item(s), {} remaining...'.format(
+                    source_dataset_chunk.count(), total_count
+                ))
 
-                    for current in source_dataset_chunk:
-                        current.save(using=self.POSTGRESQL_DB_KEY)
-
-                    total_count -= CHUNK_SIZE
+                current_model.objects.bulk_create(source_dataset_chunk)
+                total_count -= CHUNK_SIZE
 
         print(' -------- Done')

dsmrreader/provisioning/django/mysql_to_postgresql.py

@@ -3,19 +3,19 @@
 
 DATABASES = {
     'default': {
-        'ENGINE': 'django.db.backends.mysql',
+        'ENGINE': 'django.db.backends.postgresql',
         'NAME': 'dsmrreader',
         'USER': 'dsmrreader',
         'PASSWORD': 'dsmrreader',
         'HOST': 'localhost',
         'CONN_MAX_AGE': 60,
     },
-    'target': {
-        'ENGINE': 'django.db.backends.postgresql',
+    'source': {
+        'ENGINE': 'django.db.backends.mysql',
         'NAME': 'dsmrreader',
         'USER': 'dsmrreader',
         'PASSWORD': 'dsmrreader',
         'HOST': 'localhost',
         'CONN_MAX_AGE': 60,
-    }
+    },
 }