Update world entities storage format to be more compact

src/main/kotlin/ru/dbotthepony/kstarbound/VersionRegistry.kt

@@ -51,6 +51,10 @@ object VersionRegistry {
 		return migrate(read, name)
 	}
 
+	fun load(reader: JsonElement): JsonElement {
+		return migrate(VersionedJson.fromJson(reader))
+	}
+
 	private val adapter by lazy { Starbound.gson.getAdapter(VersionedJson::class.java) }
 
 	fun load(): Future<*> {

src/main/kotlin/ru/dbotthepony/kstarbound/io/SQLSavepoint.kt

@@ -0,0 +1,44 @@
+package ru.dbotthepony.kstarbound.io
+
+import java.io.Closeable
+import java.sql.Connection
+import java.sql.PreparedStatement
+
+class SQLSavepoint(connection: Connection, name: String) : Closeable {
+	private val begin: PreparedStatement
+	private val finish: PreparedStatement
+	private val rollback: PreparedStatement
+
+	init {
+		check('"' !in name) { "Invalid identifier: $name" }
+
+		begin = connection.prepareStatement("""
+			SAVEPOINT "$name"
+		""".trimIndent())
+
+		finish = connection.prepareStatement("""
+			RELEASE SAVEPOINT "$name"
+		""".trimIndent())
+
+		rollback = connection. prepareStatement("""
+			ROLLBACK TO SAVEPOINT "$name"
+		""".trimIndent())
+	}
+
+	fun execute(block: () -> Unit) {
+		try {
+			begin.execute()
+			block.invoke()
+			finish.execute()
+		} catch (err: Throwable) {
+			rollback.execute()
+			throw err
+		}
+	}
+
+	override fun close() {
+		begin.close()
+		finish.close()
+		rollback.close()
+	}
+}

src/main/kotlin/ru/dbotthepony/kstarbound/io/SQLUtils.kt

@@ -1,24 +0,0 @@
-package ru.dbotthepony.kstarbound.io
-
-import java.sql.Connection
-import java.sql.PreparedStatement
-
-data class SavepointStatements(val begin: PreparedStatement, val commit: PreparedStatement, val rollback: PreparedStatement)
-
-fun Connection.createSavepoint(name: String): SavepointStatements {
-	check('"' !in name) { "Invalid identifier: $name" }
-
-	val begin = prepareStatement("""
-		SAVEPOINT "$name"
-	""".trimIndent())
-
-	val commit = prepareStatement("""
-		RELEASE SAVEPOINT "$name"
-	""".trimIndent())
-
-	val rollback = prepareStatement("""
-		ROLLBACK TO SAVEPOINT "$name"
-	""".trimIndent())
-
-	return SavepointStatements(begin, commit, rollback)
-}

src/main/kotlin/ru/dbotthepony/kstarbound/json/VersionedJson.kt

@@ -40,5 +40,9 @@ data class VersionedJson(val id: String, val version: Int?, val content: JsonEle
 
 	companion object {
 		private val adapter by lazy { Starbound.gson.getAdapter<VersionedJson>() }
+
+		fun fromJson(data: JsonElement): VersionedJson {
+			return adapter.fromJsonTree(data)
+		}
 	}
 }

src/main/kotlin/ru/dbotthepony/kstarbound/server/world/NativeLocalWorldStorage.kt

@@ -1,5 +1,6 @@
 package ru.dbotthepony.kstarbound.server.world
 
+import com.google.gson.JsonArray
 import com.google.gson.JsonObject
 import it.unimi.dsi.fastutil.ints.Int2ObjectAVLTreeMap
 import it.unimi.dsi.fastutil.io.FastByteArrayInputStream
@@ -15,12 +16,13 @@ import ru.dbotthepony.kommons.io.writeCollection
 import ru.dbotthepony.kstarbound.Starbound
 import ru.dbotthepony.kstarbound.VersionRegistry
 import ru.dbotthepony.kstarbound.defs.EntityType
-import ru.dbotthepony.kstarbound.io.createSavepoint
+import ru.dbotthepony.kstarbound.io.SQLSavepoint
 import ru.dbotthepony.kstarbound.json.VersionedJson
+import ru.dbotthepony.kstarbound.json.readJsonArrayInflated
 import ru.dbotthepony.kstarbound.json.readJsonElement
 import ru.dbotthepony.kstarbound.json.readJsonObjectInflated
+import ru.dbotthepony.kstarbound.json.writeJsonArrayDeflated
 import ru.dbotthepony.kstarbound.json.writeJsonElement
-import ru.dbotthepony.kstarbound.json.writeJsonObjectDeflated
 import ru.dbotthepony.kstarbound.math.vector.Vector2d
 import ru.dbotthepony.kstarbound.math.vector.Vector2i
 import ru.dbotthepony.kstarbound.util.CarriedExecutor
@@ -39,7 +41,6 @@ import java.io.File
 import java.lang.ref.Cleaner
 import java.sql.Connection
 import java.sql.DriverManager
-import java.sql.PreparedStatement
 import java.util.concurrent.CompletableFuture
 import java.util.concurrent.TimeUnit
 import java.util.zip.Deflater
@@ -96,28 +97,26 @@ class NativeLocalWorldStorage(file: File?) : WorldStorage() {
 
 			it.execute("""
 				CREATE TABLE IF NOT EXISTS "entities" (
+					"x" INTEGER NOT NULL,
+					"y" INTEGER NOT NULL,
+					"data" BLOB NOT NULL,
+					PRIMARY KEY ("x", "y")
+				)
+			""".trimIndent())
+
+			it.execute("""
+				CREATE TABLE IF NOT EXISTS "unique_entities" (
+					"unique_id" BLOB NOT NULL PRIMARY KEY,
 					-- store chunks because rules for entities belonging to specific chunk might get different over time
 					"chunkX" INTEGER NOT NULL,
 					"chunkY" INTEGER NOT NULL,
 					"x" REAL NOT NULL,
-					"y" REAL NOT NULL,
+					"y" REAL NOT NULL
-					"unique_id" VARCHAR,
-					"type" VARCHAR NOT NULL,
-					"version" INTEGER NOT NULL,
-					"data" BLOB NOT NULL
 				)
 			""".trimIndent())
 
-			// Enforce unique-ness of unique entity IDs
-			// If another entity pops-up with same unique id, then it will overwrite entity in other chunk
-			// (unless colliding entities are both loaded into world's memory, which will cause runtime exception to be thrown;
-			// and no overwrite will happen)
 			it.execute("""
-				CREATE UNIQUE INDEX IF NOT EXISTS "entities_unique_id" ON "entities" ("unique_id")
+				CREATE INDEX IF NOT EXISTS "entities_chunk_pos" ON "unique_entities" ("chunkX", "chunkY")
-			""".trimIndent())
-
-			it.execute("""
-				CREATE INDEX IF NOT EXISTS "entities_chunk_pos" ON "entities" ("chunkX", "chunkY")
 			""".trimIndent())
 		}
 
@@ -167,7 +166,7 @@ class NativeLocalWorldStorage(file: File?) : WorldStorage() {
 	}
 
 	private val readEntities = connection.prepareStatement("""
-		SELECT "type", "version", "data" FROM "entities" WHERE "chunkX" = ? AND "chunkY" = ?
+		SELECT "data" FROM "entities" WHERE "x" = ? AND "y" = ?
 	""".trimIndent())
 
 	override fun loadEntities(pos: ChunkPos): CompletableFuture<Collection<AbstractEntity>> {
@@ -178,20 +177,22 @@ class NativeLocalWorldStorage(file: File?) : WorldStorage() {
 			val entities = ArrayList<AbstractEntity>()
 
 			readEntities.executeQuery().use {
-				while (it.next()) {
+				if (it.next()) {
-					val rtype = it.getString(1)
+					val data = it.getBytes(1).readJsonArrayInflated()
-					val version = it.getInt(2)
-					val type = EntityType.entries.firstOrNull { it.storeName == rtype }
 
-					if (type != null) {
+					for (entry in data) {
-						try {
+						val versioned = VersionedJson.fromJson(entry)
-							val data = it.getBytes(3).readJsonObjectInflated()
+						val type = EntityType.entries.firstOrNull { it.storeName == versioned.id }
-							entities.add(type.fromStorage(VersionRegistry.migrate(VersionedJson(rtype, version, data)) as JsonObject))
+
-						} catch (err: Throwable) {
+						if (type != null) {
-							LOGGER.error("Unable to deserialize entity in chunk $pos", err)
+							try {
+								entities.add(type.fromStorage(VersionRegistry.migrate(versioned) as JsonObject))
+							} catch (err: Throwable) {
+								LOGGER.error("Unable to deserialize entity in chunk $pos", err)
+							}
+						} else {
+							LOGGER.error("Unknown entity type ${versioned.id} in chunk $pos")
 						}
-					} else {
-						LOGGER.error("Unknown entity type $rtype in chunk $pos")
 					}
 				}
 			}
@@ -240,57 +241,53 @@ class NativeLocalWorldStorage(file: File?) : WorldStorage() {
 		}
 	}
 
-	private val clearEntities = connection.prepareStatement("""
+	private val clearUniqueEntities = connection.prepareStatement("""
-		DELETE FROM "entities" WHERE "chunkX" = ? AND "chunkY" = ?
+		DELETE FROM "unique_entities" WHERE "chunkX" = ? AND "chunkY" = ?
 	""".trimIndent())
 
-	private val beginSaveEntities: PreparedStatement
+	private val entitiesSavepoint = SQLSavepoint(connection, "save_entities")
-	private val finishSaveEntities: PreparedStatement
-	private val rollbackSaveEntities: PreparedStatement
 
-	init {
+	private val writeEntities = connection.prepareStatement("""
-		val (begin, commit, rollback) = connection.createSavepoint("save_entities")
+		REPLACE INTO "entities" ("x", "y", "data")
-		beginSaveEntities = begin
+			VALUES (?, ?, ?)
-		finishSaveEntities = commit
+	""".trimIndent())
-		rollbackSaveEntities = rollback
-	}
 
-	private val writeEntity = connection.prepareStatement("""
+	private val writeUniqueEntity = connection.prepareStatement("""
-		REPLACE INTO "entities" ("chunkX", "chunkY", "x", "y", "unique_id", "type", "version", "data")
+		REPLACE INTO "unique_entities" ("unique_id", "chunkX", "chunkY", "x", "y")
-			VALUES (?, ?, ?, ?, ?, ?, ?, ?)
+			VALUES (?, ?, ?, ?, ?)
 	""".trimIndent())
 
 	override fun saveEntities(pos: ChunkPos, entities: Collection<AbstractEntity>) {
 		executor.execute {
-			beginSaveEntities.execute()
+			entitiesSavepoint.execute {
+				clearUniqueEntities.setInt(1, pos.x)
+				clearUniqueEntities.setInt(2, pos.y)
+				clearUniqueEntities.execute()
 
-			try {
+				val storeData = JsonArray()
-				clearEntities.setInt(1, pos.x)
-				clearEntities.setInt(2, pos.y)
-				clearEntities.execute()
 
 				for (entity in entities) {
 					Starbound.storeJson {
 						val data = JsonObject()
 						entity.serialize(data)
+						storeData.add(VersionRegistry.make(entity.type.storeName, data).toJson())
 
-						writeEntity.setInt(1, pos.x)
+						if (entity.uniqueID.get() != null) {
-						writeEntity.setInt(2, pos.y)
+							writeUniqueEntity.setString(1, entity.uniqueID.get())
-						writeEntity.setDouble(3, entity.position.x)
+							writeUniqueEntity.setInt(2, pos.x)
-						writeEntity.setDouble(4, entity.position.y)
+							writeUniqueEntity.setInt(3, pos.y)
-						writeEntity.setString(5, entity.uniqueID.get())
+							writeUniqueEntity.setDouble(4, entity.position.x)
-						writeEntity.setString(6, entity.type.storeName)
+							writeUniqueEntity.setDouble(5, entity.position.y)
-						writeEntity.setInt(7, VersionRegistry.currentVersion(entity.type.storeName))
-						writeEntity.setBytes(8, data.writeJsonObjectDeflated())
 
-						writeEntity.execute()
+							writeUniqueEntity.execute()
+						}
 					}
 				}
 
-				finishSaveEntities.execute()
+				writeEntities.setInt(1, pos.x)
-			} catch (err: Throwable) {
+				writeEntities.setInt(2, pos.y)
-				rollbackSaveEntities.execute()
+				writeEntities.setBytes(3, storeData.writeJsonArrayDeflated())
-				throw err
+				writeEntities.execute()
 			}
 		}
 	}
@@ -430,7 +427,7 @@ class NativeLocalWorldStorage(file: File?) : WorldStorage() {
 	}
 
 	private val findUniqueEntity = connection.prepareStatement("""
-		SELECT "chunkX", "chunkY", "x", "y" FROM "entities" WHERE "unique_id" = ?
+		SELECT "chunkX", "chunkY", "x", "y" FROM "unique_entities" WHERE "unique_id" = ?
 	""".trimIndent())
 
 	override fun findUniqueEntity(identifier: String): CompletableFuture<UniqueEntitySearchResult?> {