diff --git a/docs/conf.py b/docs/conf.py index ffaa0aa5e..3d7b3d71b 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -56,9 +56,9 @@ # built documents. # # The short X.Y version. -version = '1.5' +version = '1.x' # The full version, including alpha/beta/rc tags. -release = '1.5.0' +release = '1.x' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/docs/faq.rst b/docs/faq.rst index 353debc3f..552d78e54 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -189,6 +189,8 @@ Only want to restore the database? - Stop the application first with ``sudo supervisorctl stop all``. This will disconnect it from the database as well. +- Importing the data could take a long time. It took MySQL 15 minutes to import nearly 3 million readings, from a compressed backup, on a RaspberryPi 3. + For **PostgreSQL** restores:: sudo sudo -u postgres dropdb dsmrreader diff --git a/docs/installation.rst b/docs/installation.rst index 4ab9630de..e33a01bb9 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -121,7 +121,7 @@ Install MariaDB. You can also choose to install the closed source MySQL, as they - Create database:: - sudo mysqladmin create dsmrreader + sudo mysqladmin --defaults-file=/etc/mysql/debian.cnf create dsmrreader - Create database user:: diff --git a/docs/locale/nl/LC_MESSAGES/faq.mo b/docs/locale/nl/LC_MESSAGES/faq.mo index 28a578a1e..36dbcf308 100644 Binary files a/docs/locale/nl/LC_MESSAGES/faq.mo and b/docs/locale/nl/LC_MESSAGES/faq.mo differ diff --git a/docs/locale/nl/LC_MESSAGES/faq.po b/docs/locale/nl/LC_MESSAGES/faq.po index b4be51ed1..de1efe7c5 100644 --- a/docs/locale/nl/LC_MESSAGES/faq.po +++ b/docs/locale/nl/LC_MESSAGES/faq.po @@ -8,8 +8,8 @@ msgid "" msgstr "" "Project-Id-Version: DSMR Reader v1.x\n" "Report-Msgid-Bugs-To: Dennis Siemensma \n" -"POT-Creation-Date: 2017-01-07 17:54+0100\n" -"PO-Revision-Date: 2017-01-07 17:56+0100\n" +"POT-Creation-Date: 2017-01-08 00:13+0100\n" +"PO-Revision-Date: 2017-01-08 00:14+0100\n" "Last-Translator: Dennis Siemensma \n" "Language-Team: Dennis Siemensma \n" "MIME-Version: 1.0\n" @@ -459,18 +459,27 @@ msgstr "" "ook voor dat de databaseverbinding van de applicatie verdwijnt." #: ../../faq.rst:192 +msgid "" +"Importing the data could take a long time. It took MySQL 15 minutes to import " +"nearly 3 million readings, from a compressed backup, on a RaspberryPi 3." +msgstr "" +"Het importeren van de gegevens kan enige tijd duren. MySQL doet er ongeveer 15 " +"minuten over om bijna 3 miljoen metingen in te lezen op een RaspberryPi 3, " +"vanuit een gecomprimeerde back-up." + +#: ../../faq.rst:194 msgid "For **PostgreSQL** restores::" msgstr "Voor herstellen van **PostgreSQL**::" -#: ../../faq.rst:206 +#: ../../faq.rst:208 msgid "For **MySQL** restores::" msgstr "Voor herstellen van **MySQL**::" -#: ../../faq.rst:220 +#: ../../faq.rst:222 msgid "Start the application again with ``sudo supervisorctl start all``." msgstr "Start de applicatie weer met ``sudo supervisorctl start all``." -#: ../../faq.rst:224 +#: ../../faq.rst:226 msgid "" "In case the version differs, you can try forcing a deployment reload by: ``sudo " "su - dsmr`` and then executing ``./post-deploy.sh``." @@ -479,11 +488,11 @@ msgstr "" "herladen. Log in met: ``sudo su - dsmr`` en voer vervolgens ``./post-deploy." "sh`` uit." -#: ../../faq.rst:228 +#: ../../faq.rst:230 msgid "How can I migrate from MySQL to PostgreSQL?" msgstr "Hoe kan ik migreren van MySQL naar PostgreSQL?" -#: ../../faq.rst:232 +#: ../../faq.rst:234 msgid "" "This is only relevant for users still running on MySQL since DSMR-reader " "``v1.5``." @@ -491,7 +500,7 @@ msgstr "" "Dit is alleen van toepassing op gebruikers van DSMR-reader vanaf ``v1.5`` in " "combinatie met MySQL." -#: ../../faq.rst:234 +#: ../../faq.rst:236 msgid "" "Start by :doc:`installing PostgreSQL as documented in chapter 1 in the " "installation guide`." @@ -499,7 +508,7 @@ msgstr "" "Start met :doc:`PostgreSQL te installeren, zoals beschreven in hoofdstuk 1 van " "de installatiehandleiding.`." -#: ../../faq.rst:236 +#: ../../faq.rst:238 msgid "" "Make sure you have enough diskspace before migrating. The following command " "will display the size of the MySQL database in MB. Execute it as ``root`` / " @@ -509,11 +518,11 @@ msgstr "" "volgende command toont de grootte van de MySQL-database, in MB. Voer dit uit " "als ``root`` / ``sudo`` gebruiker::" -#: ../../faq.rst:242 +#: ../../faq.rst:244 msgid "View the available disk space on your system::" msgstr "Bekijk hoeveel schijfruimte er beschikbaar is::" -#: ../../faq.rst:247 +#: ../../faq.rst:249 msgid "" "This will display something similar as below and you'd (most likely) need to " "check the mount on ``/`` (indicated by ``<<<<<<<<<<`` in the example)::" @@ -521,7 +530,7 @@ msgstr "" "Dit toont een soortgelijke output als hieronder. Je zult waarschijnlijk moeten " "kijken naar de mount op ``/`` (in het voorbeeld gemarkeerd met ``<<<<<<<<<<``)::" -#: ../../faq.rst:263 +#: ../../faq.rst:265 msgid "" "In the example above the mount on ``/`` has ``11GB`` available. Do not continue " "when you're not sure whether you have enough disk space available." @@ -530,20 +539,20 @@ msgstr "" "beschikbaar. Ga niet verder wanneer je niet zeker weet of je voldoende vrije " "schijfruimte beschikbaar hebt." -#: ../../faq.rst:266 +#: ../../faq.rst:268 msgid "Login as ``dsmr`` user::" msgstr "Login als ``dsmr`` gebruiker::" -#: ../../faq.rst:271 +#: ../../faq.rst:273 msgid "Now install the PostgreSQL connection client::" msgstr "Installeer nu de PostgreSQL client::" -#: ../../faq.rst:276 +#: ../../faq.rst:278 msgid "Initialize the (empty) PostgreSQL database you've just created::" msgstr "" "Initializeer de (lege) PostgreSQL-database die je zojuist hebt aangemaakt::" -#: ../../faq.rst:285 +#: ../../faq.rst:287 msgid "" "Now comes the waiting part. Please note that this **could take quite some " "time**, depending on how much data you have and the hardware used." @@ -552,7 +561,7 @@ msgstr "" "beslag nemen**, afhankelijk van hoeveel gegevens er zijn en op welke hardware " "dit draait." -#: ../../faq.rst:287 +#: ../../faq.rst:289 msgid "" "However, you can run the migration simultaneously and multiple times, while " "keeping your source database collecting data. The migration command (the second " @@ -562,7 +571,7 @@ msgstr "" "applicatie laten draaien. Het migratie-command (de tweede hieronder) gaat " "namelijk verder waar die is gebleven." -#: ../../faq.rst:292 +#: ../../faq.rst:294 msgid "" "After it's first completion, you should run the " "``dsmr_migrate_mysql_data_to_postgresql`` command one more time, to copy any " @@ -572,23 +581,23 @@ msgstr "" "aan te raden deze nogmaals uit te voeren. Hiermee kopieer je recente gegevens " "mee over." -#: ../../faq.rst:294 +#: ../../faq.rst:296 msgid "Migrate data::" msgstr "Migreer gegevens::" -#: ../../faq.rst:303 +#: ../../faq.rst:305 msgid "When completed, it's time to switch the application config to PostgreSQL::" msgstr "Wanneer dit klaar is, kun je de applicatie wisselen naar PostgreSQL::" -#: ../../faq.rst:308 +#: ../../faq.rst:310 msgid "And now reload the application to apply changes::" msgstr "En herlaad de applicatie om de wijzigingen door te voeren::" -#: ../../faq.rst:313 +#: ../../faq.rst:315 msgid "You should now be running on PostgreSQL." msgstr "Je bent nu over naar PostgreSQL." -#: ../../faq.rst:315 +#: ../../faq.rst:317 msgid "" "Please check the Status page whether data is being processed properly. The page " "also displays whether you are now running on PostgreSQL (indicated after the " @@ -598,7 +607,7 @@ msgstr "" "geeft ook aan of je nu op PostgreSQL draait (staat achter het versienummer op " "de pagina)." -#: ../../faq.rst:316 +#: ../../faq.rst:318 msgid "" "You should create or keep a **final backup of the MySQL database** before " "shutting it down and removing it's data." @@ -606,7 +615,7 @@ msgstr "" "Het is aan te raden om een **laatste back-up van de MySQL-database** te maken " "voordat je deze uitzet en compleet verwijdert." -#: ../../faq.rst:318 +#: ../../faq.rst:320 msgid "" "Due to technical reasons, any users created for the admin interface are **not** " "migrated. You can easily create a new user with::" @@ -614,7 +623,7 @@ msgstr "" "Omwille technische redenen, worden gebruikers van de admin-interface **niet** " "gemigreerd. Je kunt ze echter weer vrij eenvoudig aanmaken met::" -#: ../../faq.rst:325 +#: ../../faq.rst:327 msgid "" "In case anything went wrong, you can still easily go back to MySQL by restoring " "the config and reloading the application::" @@ -622,7 +631,7 @@ msgstr "" "Bij grote tegenvallers kun je altijd gemakkelijk terugwisselen naar MySQL. Zet " "dan de config terug en herlaad de applicatie::" -#: ../../faq.rst:334 +#: ../../faq.rst:336 msgid "" "**The next step below is irreversible if you do not have a backup of the MySQL " "data!**" @@ -630,7 +639,7 @@ msgstr "" "**De volgende stap is onomkeerbaar wanneer je geen back-up hebt van de MySQL-" "data!**" -#: ../../faq.rst:336 +#: ../../faq.rst:338 msgid "" "If you'd like to wipe MySQL from the system, go back to ``sudo`` / ``root`` " "user and execute::" @@ -638,23 +647,18 @@ msgstr "" "Wanneer je MySQL (inclusief data) helemaal wilt verwijderen, voer dan het " "volgende uit als ``sudo`` / ``root`` gebruiker::" -#: ../../faq.rst:344 +#: ../../faq.rst:346 msgid "Feature/bug report" msgstr "Verzoek/fout melden" -#: ../../faq.rst:345 +#: ../../faq.rst:347 msgid "*How can I propose a feature or report a bug I've found?*" msgstr "*Hoe kan ik een verzoek indienen of een fout melden?*" -#: ../../faq.rst:349 +#: ../../faq.rst:351 msgid "" "`Just create a ticket at Github `_." msgstr "" "`Maak een ticket aan op Github `_." - -#~ msgid "Please check the Status page whether data is being processed properly." -#~ msgstr "" -#~ "Controleer op de Status-pagina of de gegevens nog steeds goed verwerkt " -#~ "worden." diff --git a/dsmr_backend/management/commands/dsmr_migrate_mysql_data_to_postgresql.py b/dsmr_backend/management/commands/dsmr_migrate_mysql_data_to_postgresql.py index a3fa74922..8a424747a 100644 --- a/dsmr_backend/management/commands/dsmr_migrate_mysql_data_to_postgresql.py +++ b/dsmr_backend/management/commands/dsmr_migrate_mysql_data_to_postgresql.py @@ -1,7 +1,7 @@ from django.core.management.base import BaseCommand, CommandError from django.utils.translation import ugettext as _ from django.db.utils import OperationalError, ConnectionDoesNotExist -from django.db import connections, transaction +from django.db import connections from dsmr_consumption.models.consumption import ElectricityConsumption, GasConsumption from dsmr_backup.models.settings import DropboxSettings, BackupSettings @@ -24,8 +24,8 @@ class Command(BaseCommand): help = _('Migrates the data inside a MySQL database of this project to a PostgreSQL database.') # As defined in migration settings. - MYSQL_DB_KEY = 'default' - POSTGRESQL_DB_KEY = 'target' + MYSQL_DB_KEY = 'source' + POSTGRESQL_DB_KEY = 'default' OVERRIDE_MODELS = ( APISettings, BackupSettings, DropboxSettings, ConsumptionSettings, DataloggerSettings, FrontendSettings, @@ -86,6 +86,7 @@ def _migrate_resume(self): for current_model in self.APPEND_MODELS: source_dataset = current_model.objects.using(self.MYSQL_DB_KEY).all().order_by('pk') + print() print(' - {}'.format(current_model.__name__)) try: @@ -98,20 +99,17 @@ def _migrate_resume(self): source_dataset = source_dataset.filter(pk__gt=latest_at_target.pk) # Chunk to indicate progress and keep memory usage low. - CHUNK_SIZE = 5000 + CHUNK_SIZE = 1000 total_count = source_dataset.count() print(' --- Total items to copy: {}'.format(total_count)) - with transaction.atomic(using=self.POSTGRESQL_DB_KEY): - for start in range(0, total_count, CHUNK_SIZE): - source_dataset_chunk = source_dataset[start:start + CHUNK_SIZE] - print(' --- Processing {} item(s), {} remaining...'.format( - source_dataset_chunk.count(), total_count - )) + for start in range(0, total_count, CHUNK_SIZE): + source_dataset_chunk = source_dataset[start:start + CHUNK_SIZE] + print(' --- Processing {} item(s), {} remaining...'.format( + source_dataset_chunk.count(), total_count + )) - for current in source_dataset_chunk: - current.save(using=self.POSTGRESQL_DB_KEY) - - total_count -= CHUNK_SIZE + current_model.objects.bulk_create(source_dataset_chunk) + total_count -= CHUNK_SIZE print(' -------- Done') diff --git a/dsmrreader/provisioning/django/mysql_to_postgresql.py b/dsmrreader/provisioning/django/mysql_to_postgresql.py index b14b60cf4..a45889c82 100644 --- a/dsmrreader/provisioning/django/mysql_to_postgresql.py +++ b/dsmrreader/provisioning/django/mysql_to_postgresql.py @@ -3,19 +3,19 @@ DATABASES = { 'default': { - 'ENGINE': 'django.db.backends.mysql', + 'ENGINE': 'django.db.backends.postgresql', 'NAME': 'dsmrreader', 'USER': 'dsmrreader', 'PASSWORD': 'dsmrreader', 'HOST': 'localhost', 'CONN_MAX_AGE': 60, }, - 'target': { - 'ENGINE': 'django.db.backends.postgresql', + 'source': { + 'ENGINE': 'django.db.backends.mysql', 'NAME': 'dsmrreader', 'USER': 'dsmrreader', 'PASSWORD': 'dsmrreader', 'HOST': 'localhost', 'CONN_MAX_AGE': 60, - } + }, }