Skip to content
Permalink
Browse files

Implement mongodb migration as cli feature

Signed-off-by: Till Kottmann <me@deletescape.ch>
  • Loading branch information
deletescape committed Oct 6, 2019
1 parent b086092 commit d9ca44165354ebe8003cbc3bc0b0616994ffae71
@@ -13,6 +13,7 @@ dependencies {
compile("org.jetbrains.kotlin:kotlin-stdlib-jdk8:$kotlin_version")

compile(project(":data:base"))
compile(project(":data:migration"))
compile(project(":commons"))
}

@@ -7,7 +7,8 @@ class FeatureManager {
EnvironmentHelperFeature(),
InitStoreFeature(),
UserFeature(),
DocumentFeature()
DocumentFeature(),
MigationFeature()
)

fun evaluate(command: String) {
@@ -0,0 +1,49 @@
package dog.del.cli.features

import com.mongodb.MongoCredential
import dog.del.cli.DogbinCli
import dog.del.data.migration.MongoMigration

class MigationFeature : CliFeature {
override val metadata = FeatureMetadata(
listOf("migrate"),
listOf(
arg(
"host",
false,
"The host on which mongodb is running. Defaults to localhost."
),
arg(
"db",
false,
"The name of the db. Defaults to dogbin_dev."
),
arg(
"username",
false
),
arg(
"password",
false
)
),
mapOf(
null to "Migrate from MongDb to the new Xodus backend"
)
)

override fun execute(name: String, args: Map<String, String?>) {
val host = args["host"] ?: "localhost"
val dbName = args["db"] ?: "dogbin_dev"
val username = args["username"]
val password = args["password"]
val credential = if (username != null && password != null) MongoCredential.createCredential(username, dbName, password.toCharArray()) else null

MongoMigration(
xdStore = DogbinCli.Companion.Globals.getStore(),
mongoHost = host,
dbName = dbName,
credential = credential
).migrate()
}
}
@@ -0,0 +1,8 @@
package dog.del.commons

import kotlin.math.round

fun Double.roundToDecimals(numDecimalPlaces: Int): Double {
val factor = Math.pow(10.0, numDecimalPlaces.toDouble())
return round(this * factor) / factor
}
@@ -27,6 +27,8 @@ class XdUser(entity: Entity) : XdEntity(entity), User<XdUserRole> {

fun find(username: String) = filter { it.username eq username }.firstOrNull()

fun findOrNew(username: String, init: XdUser.() -> Unit) = findOrNew(filter { it.username eq username }, init)

/**
* User used for pastes created via anonymous api
*/
@@ -76,6 +78,13 @@ class XdUser(entity: Entity) : XdEntity(entity), User<XdUserRole> {
default = { _, _ -> Date.getInstance() }
)

/**
* ONLY ever use this for migration purposes
*/
fun setPasswordHashed(password: String) {
_password = password
}

override fun checkPassword(password: String) =
this._password != null && Password.verify(password, this._password!!).verified
}
@@ -0,0 +1,18 @@
val kotlin_version: String by rootProject

plugins {
kotlin("kapt")
}

dependencies {
compile("org.jetbrains.kotlin:kotlin-stdlib-jdk8:$kotlin_version")

compile("org.litote.kmongo:kmongo:3.11.0")
kapt("org.litote.kmongo:kmongo-annotation-processor:3.11.0")

compile(project(":data:base"))
compile(project(":commons"))
}

kotlin.sourceSets["main"].kotlin.srcDirs("src")
kotlin.sourceSets["test"].kotlin.srcDirs("test")
@@ -0,0 +1,86 @@
package dog.del.data.migration

import com.mongodb.MongoCredential
import com.mongodb.ServerAddress
import dog.del.commons.date
import dog.del.commons.isUrl
import dog.del.commons.roundToDecimals
import dog.del.data.base.model.document.XdDocument
import dog.del.data.base.model.document.XdDocumentType
import dog.del.data.base.model.user.XdUser
import dog.del.data.base.model.user.XdUserRole
import dog.del.data.migration.model.MongoDocument
import dog.del.data.migration.model.User
import jetbrains.exodus.database.TransientEntityStore
import kotlinx.dnq.creator.findOrNew
import org.bson.types.ObjectId
import org.litote.kmongo.*

class MongoMigration(
val xdStore: TransientEntityStore,
mongoHost: String = "localhost",
dbName: String = "dogbin_dev",
credential: MongoCredential? = null
) {
private val client = KMongo.createClient(ServerAddress(mongoHost), listOfNotNull(credential))
private val database = client.getDatabase(dbName)
private val mongoDocuments = database.getCollectionOfName<MongoDocument>("mongo_document")
private val users = database.getCollection<User>()

private var userCount = 0
private var docCount = 0

fun migrate() {
// Reset counters
userCount = 0
docCount = 0

val count = mongoDocuments.countDocuments()
println("Starting migration of $count documents")
mongoDocuments.find().forEachIndexed { i, it ->
try {
migrateDocument(it)
print("Migrated ${it._id} successfully")
} catch (e: Exception) {
print("Failed to migrate ${it._id}: ${e.message}")
}
val percent = ((i + 1 / count.toDouble()) * 100).roundToDecimals(2)
println(" ${i + 1}/$count ($percent%) ")
}
println("Done! Migrated $docCount documents and $userCount users")
}

private fun migrateDocument(document: MongoDocument): XdDocument = xdStore.transactional {
XdDocument.findOrNew(document._id) {
stringContent = document.content
type = if (document.content.isUrl()) XdDocumentType.URL else XdDocumentType.PASTE
version = document.version
viewCount = document.viewCount
owner = migrateUser(findUser(document.owner))
created = date()
docCount++
}
}

private fun findUser(id: Id<User>) = users.find("{'_id':ObjectId(\"$id\")}").first()

private fun migrateUser(user: User): XdUser = xdStore.transactional {
// Use api anon for all existing anonymous users as sessions will not be preserved anyways
if (user.is_anonymous) {
XdUser.apiAnon
} else {
// Only migrate once and return existing user otherwise
XdUser.findOrNew(user.username) {
password = String(user.password.data)
created = date()
role = when {
user.is_anonymous -> XdUserRole.ANON
user.is_system -> XdUserRole.SYSTEM
"admin" in user.roles -> XdUserRole.ADMIN
else -> XdUserRole.USER
}
userCount++
}
}
}
}
@@ -0,0 +1,14 @@
package dog.del.data.migration.model

import org.litote.kmongo.Data
import org.litote.kmongo.Id

@Data()
data class MongoDocument(
val _id: String,
val isUrl: Boolean,
val content: String,
val viewCount: Int,
val version: Int,
val owner: Id<User>
)
@@ -0,0 +1,18 @@
package dog.del.data.migration.model

import org.bson.codecs.pojo.annotations.BsonId
import org.bson.types.Binary
import org.litote.kmongo.Data
import org.litote.kmongo.Id

@Data()
data class User(
val _id: Id<User>,
val username: String,
val password: Binary,
val is_active: Boolean,
val is_anonymous: Boolean,
val is_system: Boolean,
val is_authenticated: Boolean,
val roles: List<String>
)
@@ -1 +1 @@
include("model", "base")
include("model", "migration", "base")
@@ -8,4 +8,4 @@ pluginManagement {

rootProject.name = "dogbin"

include("app", "pebble", "commons", "data", "data:model", "data:base", "cli")
include("app", "pebble", "commons", "data", "data:model", "data:migration", "data:base", "cli")

0 comments on commit d9ca441

Please sign in to comment.
You can’t perform that action at this time.