DBot/KStarbound
DBot/KStarbound
3
0
Fork 0
Code Issues 1 Pull Requests Packages Projects Releases Wiki Activity

Update world entities storage format to be more compact

Browse Source
This commit is contained in:
DBotThePony 2024-08-11 00:15:36 +07:00
parent ebd2dd3ab1
commit 4bd85c0e0e
Signed by: DBot
GPG Key ID: DCC23B5715498507
5 changed files with 112 additions and 87 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
src/main/kotlin/ru/dbotthepony/kstarbound/VersionRegistry.kt
View File

@ -51,6 +51,10 @@ object VersionRegistry {
return migrate(read, name)
}
fun load(reader: JsonElement): JsonElement {
return migrate(VersionedJson.fromJson(reader))
}
private val adapter by lazy { Starbound.gson.getAdapter(VersionedJson::class.java) }
fun load(): Future<*> {

44
src/main/kotlin/ru/dbotthepony/kstarbound/io/SQLSavepoint.kt Normal file
View File

@ -0,0 +1,44 @@
package ru.dbotthepony.kstarbound.io
import java.io.Closeable
import java.sql.Connection
import java.sql.PreparedStatement
class SQLSavepoint(connection: Connection, name: String) : Closeable {
private val begin: PreparedStatement
private val finish: PreparedStatement
private val rollback: PreparedStatement
init {
check('"' !in name) { "Invalid identifier: $name" }
begin = connection.prepareStatement("""
SAVEPOINT "$name"
""".trimIndent())
finish = connection.prepareStatement("""
RELEASE SAVEPOINT "$name"
""".trimIndent())
rollback = connection. prepareStatement("""
ROLLBACK TO SAVEPOINT "$name"
""".trimIndent())
}
fun execute(block: () -> Unit) {
try {
begin.execute()
block.invoke()
finish.execute()
} catch (err: Throwable) {
rollback.execute()
throw err
}
}
override fun close() {
begin.close()
finish.close()
rollback.close()
}
}

24
src/main/kotlin/ru/dbotthepony/kstarbound/io/SQLUtils.kt
View File

@ -1,24 +0,0 @@
package ru.dbotthepony.kstarbound.io
import java.sql.Connection
import java.sql.PreparedStatement
data class SavepointStatements(val begin: PreparedStatement, val commit: PreparedStatement, val rollback: PreparedStatement)
fun Connection.createSavepoint(name: String): SavepointStatements {
check('"' !in name) { "Invalid identifier: $name" }
val begin = prepareStatement("""
SAVEPOINT "$name"
""".trimIndent())
val commit = prepareStatement("""
RELEASE SAVEPOINT "$name"
""".trimIndent())
val rollback = prepareStatement("""
ROLLBACK TO SAVEPOINT "$name"
""".trimIndent())
return SavepointStatements(begin, commit, rollback)
}

4
src/main/kotlin/ru/dbotthepony/kstarbound/json/VersionedJson.kt
View File

@ -40,5 +40,9 @@ data class VersionedJson(val id: String, val version: Int?, val content: JsonEle
companion object {
private val adapter by lazy { Starbound.gson.getAdapter<VersionedJson>() }
fun fromJson(data: JsonElement): VersionedJson {
return adapter.fromJsonTree(data)
}
}
}

123
src/main/kotlin/ru/dbotthepony/kstarbound/server/world/NativeLocalWorldStorage.kt
View File

@ -1,5 +1,6 @@
package ru.dbotthepony.kstarbound.server.world
import com.google.gson.JsonArray
import com.google.gson.JsonObject
import it.unimi.dsi.fastutil.ints.Int2ObjectAVLTreeMap
import it.unimi.dsi.fastutil.io.FastByteArrayInputStream
@ -15,12 +16,13 @@ import ru.dbotthepony.kommons.io.writeCollection
import ru.dbotthepony.kstarbound.Starbound
import ru.dbotthepony.kstarbound.VersionRegistry
import ru.dbotthepony.kstarbound.defs.EntityType
import ru.dbotthepony.kstarbound.io.createSavepoint
import ru.dbotthepony.kstarbound.io.SQLSavepoint
import ru.dbotthepony.kstarbound.json.VersionedJson
import ru.dbotthepony.kstarbound.json.readJsonArrayInflated
import ru.dbotthepony.kstarbound.json.readJsonElement
import ru.dbotthepony.kstarbound.json.readJsonObjectInflated
import ru.dbotthepony.kstarbound.json.writeJsonArrayDeflated
import ru.dbotthepony.kstarbound.json.writeJsonElement
import ru.dbotthepony.kstarbound.json.writeJsonObjectDeflated
import ru.dbotthepony.kstarbound.math.vector.Vector2d
import ru.dbotthepony.kstarbound.math.vector.Vector2i
import ru.dbotthepony.kstarbound.util.CarriedExecutor
@ -39,7 +41,6 @@ import java.io.File
import java.lang.ref.Cleaner
import java.sql.Connection
import java.sql.DriverManager
import java.sql.PreparedStatement
import java.util.concurrent.CompletableFuture
import java.util.concurrent.TimeUnit
import java.util.zip.Deflater
@ -96,28 +97,26 @@ class NativeLocalWorldStorage(file: File?) : WorldStorage() {
it.execute("""
CREATE TABLE IF NOT EXISTS "entities" (
"x" INTEGER NOT NULL,
"y" INTEGER NOT NULL,
"data" BLOB NOT NULL,
PRIMARY KEY ("x", "y")
)
""".trimIndent())
it.execute("""
CREATE TABLE IF NOT EXISTS "unique_entities" (
"unique_id" BLOB NOT NULL PRIMARY KEY,
-- store chunks because rules for entities belonging to specific chunk might get different over time
"chunkX" INTEGER NOT NULL,
"chunkY" INTEGER NOT NULL,
"x" REAL NOT NULL,
"y" REAL NOT NULL,
"unique_id" VARCHAR,
"type" VARCHAR NOT NULL,
"version" INTEGER NOT NULL,
"data" BLOB NOT NULL
"y" REAL NOT NULL
)
""".trimIndent())
// Enforce unique-ness of unique entity IDs
// If another entity pops-up with same unique id, then it will overwrite entity in other chunk
// (unless colliding entities are both loaded into world's memory, which will cause runtime exception to be thrown;
// and no overwrite will happen)
it.execute("""
CREATE UNIQUE INDEX IF NOT EXISTS "entities_unique_id" ON "entities" ("unique_id")
""".trimIndent())
it.execute("""
CREATE INDEX IF NOT EXISTS "entities_chunk_pos" ON "entities" ("chunkX", "chunkY")
CREATE INDEX IF NOT EXISTS "entities_chunk_pos" ON "unique_entities" ("chunkX", "chunkY")
""".trimIndent())
}
@ -167,7 +166,7 @@ class NativeLocalWorldStorage(file: File?) : WorldStorage() {
}
private val readEntities = connection.prepareStatement("""
SELECT "type", "version", "data" FROM "entities" WHERE "chunkX" = ? AND "chunkY" = ?
SELECT "data" FROM "entities" WHERE "x" = ? AND "y" = ?
""".trimIndent())
override fun loadEntities(pos: ChunkPos): CompletableFuture<Collection<AbstractEntity>> {
@ -178,20 +177,22 @@ class NativeLocalWorldStorage(file: File?) : WorldStorage() {
val entities = ArrayList<AbstractEntity>()
readEntities.executeQuery().use {
while (it.next()) {
val rtype = it.getString(1)
val version = it.getInt(2)
val type = EntityType.entries.firstOrNull { it.storeName == rtype }
if (it.next()) {
val data = it.getBytes(1).readJsonArrayInflated()
if (type != null) {
try {
val data = it.getBytes(3).readJsonObjectInflated()
entities.add(type.fromStorage(VersionRegistry.migrate(VersionedJson(rtype, version, data)) as JsonObject))
} catch (err: Throwable) {
LOGGER.error("Unable to deserialize entity in chunk $pos", err)
for (entry in data) {
val versioned = VersionedJson.fromJson(entry)
val type = EntityType.entries.firstOrNull { it.storeName == versioned.id }
if (type != null) {
try {
entities.add(type.fromStorage(VersionRegistry.migrate(versioned) as JsonObject))
} catch (err: Throwable) {
LOGGER.error("Unable to deserialize entity in chunk $pos", err)
}
} else {
LOGGER.error("Unknown entity type ${versioned.id} in chunk $pos")
}
} else {
LOGGER.error("Unknown entity type $rtype in chunk $pos")
}
}
}
@ -240,57 +241,53 @@ class NativeLocalWorldStorage(file: File?) : WorldStorage() {
}
}
private val clearEntities = connection.prepareStatement("""
DELETE FROM "entities" WHERE "chunkX" = ? AND "chunkY" = ?
private val clearUniqueEntities = connection.prepareStatement("""
DELETE FROM "unique_entities" WHERE "chunkX" = ? AND "chunkY" = ?
""".trimIndent())
private val beginSaveEntities: PreparedStatement
private val finishSaveEntities: PreparedStatement
private val rollbackSaveEntities: PreparedStatement
private val entitiesSavepoint = SQLSavepoint(connection, "save_entities")
init {
val (begin, commit, rollback) = connection.createSavepoint("save_entities")
beginSaveEntities = begin
finishSaveEntities = commit
rollbackSaveEntities = rollback
}
private val writeEntities = connection.prepareStatement("""
REPLACE INTO "entities" ("x", "y", "data")
VALUES (?, ?, ?)
""".trimIndent())
private val writeEntity = connection.prepareStatement("""
REPLACE INTO "entities" ("chunkX", "chunkY", "x", "y", "unique_id", "type", "version", "data")
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
private val writeUniqueEntity = connection.prepareStatement("""
REPLACE INTO "unique_entities" ("unique_id", "chunkX", "chunkY", "x", "y")
VALUES (?, ?, ?, ?, ?)
""".trimIndent())
override fun saveEntities(pos: ChunkPos, entities: Collection<AbstractEntity>) {
executor.execute {
beginSaveEntities.execute()
entitiesSavepoint.execute {
clearUniqueEntities.setInt(1, pos.x)
clearUniqueEntities.setInt(2, pos.y)
clearUniqueEntities.execute()
try {
clearEntities.setInt(1, pos.x)
clearEntities.setInt(2, pos.y)
clearEntities.execute()
val storeData = JsonArray()
for (entity in entities) {
Starbound.storeJson {
val data = JsonObject()
entity.serialize(data)
storeData.add(VersionRegistry.make(entity.type.storeName, data).toJson())
writeEntity.setInt(1, pos.x)
writeEntity.setInt(2, pos.y)
writeEntity.setDouble(3, entity.position.x)
writeEntity.setDouble(4, entity.position.y)
writeEntity.setString(5, entity.uniqueID.get())
writeEntity.setString(6, entity.type.storeName)
writeEntity.setInt(7, VersionRegistry.currentVersion(entity.type.storeName))
writeEntity.setBytes(8, data.writeJsonObjectDeflated())
if (entity.uniqueID.get() != null) {
writeUniqueEntity.setString(1, entity.uniqueID.get())
writeUniqueEntity.setInt(2, pos.x)
writeUniqueEntity.setInt(3, pos.y)
writeUniqueEntity.setDouble(4, entity.position.x)
writeUniqueEntity.setDouble(5, entity.position.y)
writeEntity.execute()
writeUniqueEntity.execute()
}
}
}
finishSaveEntities.execute()
} catch (err: Throwable) {
rollbackSaveEntities.execute()
throw err
writeEntities.setInt(1, pos.x)
writeEntities.setInt(2, pos.y)
writeEntities.setBytes(3, storeData.writeJsonArrayDeflated())
writeEntities.execute()
}
}
}
@ -430,7 +427,7 @@ class NativeLocalWorldStorage(file: File?) : WorldStorage() {
}
private val findUniqueEntity = connection.prepareStatement("""
SELECT "chunkX", "chunkY", "x", "y" FROM "entities" WHERE "unique_id" = ?
SELECT "chunkX", "chunkY", "x", "y" FROM "unique_entities" WHERE "unique_id" = ?
""".trimIndent())
override fun findUniqueEntity(identifier: String): CompletableFuture<UniqueEntitySearchResult?> {