Skip to content
Permalink
Browse files

Improve/Fix mongo migration

Additionally:
 - Support letter + digit characters from all languages in slugs
 - Support some emojis and symbols in slugs

Signed-off-by: Till Kottmann <me@deletescape.ch>
  • Loading branch information
deletescape committed Nov 5, 2019
1 parent 88de657 commit 24dbe88ab20680d5a544a3c650e3e294b8c48a13
@@ -7,6 +7,7 @@ plugins {

application {
mainClassName = "dog.del.cli.DogbinCli"
applicationDefaultJvmArgs = listOf("-Xmx6g")
}

dependencies {
@@ -33,8 +33,8 @@ class DogbinCli {
private var storeEnvHash: Int = 0
private val currentStoreEnvHash get() = (storeLocation + storeEnvironment).hashCode()

var storeLocation: String = ""
var storeEnvironment: String = ""
var storeLocation: String = "dogbin.xdb"
var storeEnvironment: String = "prod"

fun getStore(): TransientEntityStore {
if (storeLocation.isNotBlank() && storeEnvironment.isNotBlank()) {
@@ -16,7 +16,7 @@ class MigationFeature : CliFeature {
arg(
"db",
false,
"The name of the db. Defaults to dogbin_dev."
"The name of the db. Defaults to dogbin."
),
arg(
"username",
@@ -34,10 +34,14 @@ class MigationFeature : CliFeature {

override fun execute(name: String, args: Map<String, String?>) {
val host = args["host"] ?: "localhost"
val dbName = args["db"] ?: "dogbin_dev"
val dbName = args["db"] ?: "dogbin"
val username = args["username"]
val password = args["password"]
val credential = if (username != null && password != null) MongoCredential.createCredential(username, dbName, password.toCharArray()) else null
val credential = if (username != null && password != null) MongoCredential.createCredential(
username,
dbName,
password.toCharArray()
) else null

MongoMigration(
xdStore = DogbinCli.Companion.Globals.getStore(),
@@ -0,0 +1,18 @@
package dog.del.commons

private val allowedBlocks = arrayOf(
Character.UnicodeBlock.MISCELLANEOUS_SYMBOLS_AND_PICTOGRAPHS,
Character.UnicodeBlock.MISCELLANEOUS_SYMBOLS_AND_ARROWS,
Character.UnicodeBlock.MISCELLANEOUS_SYMBOLS,
Character.UnicodeBlock.EMOTICONS,
Character.UnicodeBlock.TRANSPORT_AND_MAP_SYMBOLS
)

private val allowedChars = charArrayOf(
'-',
'_'
)

fun String.validSlug() = length >= 3 && codePoints().allMatch {
Character.isLetterOrDigit(it) || Character.UnicodeBlock.of(it) in allowedBlocks || it.toChar() in allowedChars
}
@@ -0,0 +1,43 @@
package dog.del.commons

import org.junit.jupiter.api.Assertions
import org.junit.jupiter.api.DynamicTest
import org.junit.jupiter.api.TestFactory

class SlugTest {
@TestFactory
fun `is valid slug`() = listOf(
"about",
"blog",
"bl_og",
"-blog",
"b-log",
"PutoAntón",
"LOVE七七",
"\uD83D\uDC36\uD83D\uDDD1",
"павло_тичина",
"hasła",
"ЕБАТЦАТОП",
"虫虫虫"
).map { slug ->
DynamicTest.dynamicTest("Is valid slug `$slug`") {
Assertions.assertTrue(slug.validSlug(), "Expected `$slug` to be detected as valid slug")
}
}

@TestFactory
fun `is invalid slug`() = listOf(
"///",
"//a",
"a",
"a a",
" ",
"\ntest",
"jquery.js",
"1+2"
).map { slug ->
DynamicTest.dynamicTest("Is invalid slug `$slug`") {
Assertions.assertFalse(slug.validSlug(), "Expected `$slug` to be detected as invalid slug")
}
}
}
@@ -1,7 +1,9 @@
package dog.del.data.base.model.document

import com.jetbrains.teamsys.dnq.database.PropertyConstraint
import dog.del.commons.Date
import dog.del.commons.date
import dog.del.commons.validSlug
import dog.del.data.base.model.user.XdUser
import dog.del.data.base.utils.xdRequiredDateProp
import dog.del.data.model.Document
@@ -27,11 +29,9 @@ class XdDocument(entity: Entity) : XdEntity(entity), Document<XdDocumentType, Xd

fun verifySlug(slug: String) = if (slug.length < 3) {
"Custom Urls need to be at least 3 characters long"
} else if (!slug.matches(slugRegex)) {
} else if (!slug.validSlug()) {
"Custom URLs must be alphanumeric and cannot contain spaces"
} else null

val slugRegex = Regex("^[\\w-]{3,}\$")
}

override fun constructor() {
@@ -41,7 +41,12 @@ class XdDocument(entity: Entity) : XdEntity(entity), Document<XdDocumentType, Xd
}

override var slug by xdRequiredStringProp(unique = true, trimmed = true) {
regex(slugRegex)
constraints.add(object : PropertyConstraint<String?>() {
override fun getExceptionMessage(propertyName: String, propertyValue: String?) =
"\"$propertyValue\" isn't a valid slug"

override fun isValid(value: String?) = value?.validSlug() == true
})
}

override var type by xdLink1(XdDocumentType)
@@ -7,9 +7,13 @@ plugins {
dependencies {
compile("org.jetbrains.kotlin:kotlin-stdlib-jdk8:$kotlin_version")

compile("org.litote.kmongo:kmongo:3.11.0")
compile("org.litote.kmongo:kmongo:3.11.1")
compile("me.tongfei:progressbar:0.7.4")
kapt("org.litote.kmongo:kmongo-annotation-processor:3.11.0")

// Disable logging from xodus
compile("org.slf4j:slf4j-nop:1.7.28")

compile(project(":data:base"))
compile(project(":commons"))
}
@@ -4,83 +4,82 @@ import com.mongodb.MongoCredential
import com.mongodb.ServerAddress
import dog.del.commons.date
import dog.del.commons.isUrl
import dog.del.commons.roundToDecimals
import dog.del.data.base.model.document.XdDocument
import dog.del.data.base.model.document.XdDocumentType
import dog.del.data.base.model.user.XdUser
import dog.del.data.base.model.user.XdUserRole
import dog.del.data.migration.model.MongoDocument
import dog.del.data.migration.model.User
import jetbrains.exodus.database.TransientEntityStore
import kotlinx.dnq.creator.findOrNew
import org.bson.types.ObjectId
import me.tongfei.progressbar.ProgressBar
import org.litote.kmongo.*

class MongoMigration(
val xdStore: TransientEntityStore,
mongoHost: String = "localhost",
dbName: String = "dogbin_dev",
dbName: String = "dogbin",
credential: MongoCredential? = null
) {
private val client = KMongo.createClient(ServerAddress(mongoHost), listOfNotNull(credential))
private val database = client.getDatabase(dbName)
private val mongoDocuments = database.getCollectionOfName<MongoDocument>("mongo_document")
private val users = database.getCollection<User>()

private var userCount = 0
private var docCount = 0

fun migrate() {
// Reset counters
userCount = 0
docCount = 0
val total = mongoDocuments.estimatedDocumentCount()
println("Starting migration of $total documents")

val pb = ProgressBar("Migration", total)
mongoDocuments.find().forEach { doc ->
val slug = doc._id.sanitize()
pb.extraMessage = slug

val count = mongoDocuments.countDocuments()
println("Starting migration of $count documents")
mongoDocuments.find().forEachIndexed { i, it ->
try {
migrateDocument(it)
print("Migrated ${it._id} successfully")
migrateDocument(doc, slug)
} catch (e: Exception) {
print("Failed to migrate ${it._id}: ${e.message}")
println("\nError while migrating \"$slug\": $e")
}
val percent = (((i + 1) / count.toDouble()) * 100).roundToDecimals(2)
println(" ${i + 1}/$count ($percent%) ")

pb.step()
}
println("Done! Migrated $docCount documents and $userCount users")
pb.close()
println("Done!")
}

private fun migrateDocument(document: MongoDocument): XdDocument = xdStore.transactional {
XdDocument.findOrNew(document._id) {
private fun migrateDocument(document: MongoDocument, slug: String) = xdStore.transactional {
XdDocument.findOrNew(slug) {
stringContent = document.content
type = if (document.content.isUrl()) XdDocumentType.URL else XdDocumentType.PASTE
version = document.version
viewCount = document.viewCount
owner = migrateUser(findUser(document.owner))
created = date()
docCount++
}
}

private fun findUser(id: Id<User>?) = if (id == null) null else users.find("{'_id':ObjectId(\"$id\")}").first()
private fun findUser(id: Id<User>?) =
if (id == null) null else users.find("{'_id':ObjectId(\"$id\")}").first()

private fun migrateUser(user: User?): XdUser = xdStore.transactional {
private fun migrateUser(user: User?): XdUser {
// Use api anon for all existing anonymous users as sessions will not be preserved anyways
if (user == null || user.is_anonymous) {
return if (user == null || user.is_anonymous) {
XdUser.apiAnon
} else {
// Only migrate once and return existing user otherwise
XdUser.findOrNew(user.username) {
password = String(user.password.data)
created = date()
role = when {
user.is_anonymous -> XdUserRole.ANON
user.is_system -> XdUserRole.SYSTEM
"admin" in user.roles -> XdUserRole.ADMIN
else -> XdUserRole.USER
}
userCount++
if (role.requiresPassword) {
password = String(user.password.data)
}
}
}
}

private fun String.sanitize(): String = this.substringBefore('.')
}
10 dgb-cli
@@ -1,11 +1,17 @@
#!/usr/bin/env sh

CURRENTDIR=$(pwd)

# Absolute path to this script,
SCRIPT=$(readlink -f "$0")
# Absolute path this script is in
SCRIPTPATH=$(dirname "$SCRIPT")

cd $SCRIPTPATH

# Build/Install
sh -c $SCRIPTPATH/gradlew :cli:installDist > /dev/null
./gradlew :cli:installDist > /dev/null
# Start the shell
sh -c $SCRIPTPATH/cli/build/install/cli/bin/cli
./cli/build/install/cli/bin/cli

cd $CURRENTDIR

0 comments on commit 24dbe88

Please sign in to comment.
You can’t perform that action at this time.