Blazingly fast system data deserialization

src/main/kotlin/ru/dbotthepony/kstarbound/Ext.kt

@@ -9,6 +9,7 @@ import com.google.gson.TypeAdapterFactory
 import com.google.gson.reflect.TypeToken
 import com.google.gson.stream.JsonReader
 import ru.dbotthepony.kommons.util.KOptional
+import ru.dbotthepony.kstarbound.json.FastJsonTreeReader
 import java.util.Arrays
 import java.util.concurrent.Callable
 import java.util.concurrent.ForkJoinPool
@@ -49,7 +50,9 @@ operator fun <K : Any, V : Any> ImmutableMap.Builder<K, V>.set(key: K, value: V)
 fun String.sintern(): String = Starbound.STRINGS.intern(this)
 
 inline fun <reified T> Gson.fromJson(reader: JsonReader): T? = fromJson<T>(reader, T::class.java)
-inline fun <reified T> Gson.fromJson(reader: JsonElement): T? = fromJson<T>(reader, T::class.java)
+inline fun <reified T> Gson.fromJson(reader: JsonElement): T? = getAdapter(T::class.java).read(FastJsonTreeReader(reader))
+
+fun <T> Gson.fromJsonFast(reader: JsonElement, type: Class<T>): T = getAdapter(type).read(FastJsonTreeReader(reader))
 
 /**
  * guarantees even distribution of tasks while also preserving encountered order of elements

src/main/kotlin/ru/dbotthepony/kstarbound/defs/world/TerrestrialWorldParameters.kt

@@ -30,6 +30,7 @@ import ru.dbotthepony.kstarbound.defs.tile.BuiltinMetaMaterials
 import ru.dbotthepony.kstarbound.defs.tile.LiquidDefinition
 import ru.dbotthepony.kstarbound.defs.tile.isEmptyLiquid
 import ru.dbotthepony.kstarbound.fromJson
+import ru.dbotthepony.kstarbound.fromJsonFast
 import ru.dbotthepony.kstarbound.io.readInternedString
 import ru.dbotthepony.kstarbound.io.readVector2d
 import ru.dbotthepony.kstarbound.io.writeStruct2d
@@ -182,7 +183,7 @@ class TerrestrialWorldParameters : VisitableWorldParameters() {
 	override fun fromJson(data: JsonObject) {
 		super.fromJson(data)
 
-		val read = Starbound.gson.fromJson(data, StoreData::class.java)
+		val read = Starbound.gson.fromJsonFast(data, StoreData::class.java)
 
 		primaryBiome = read.primaryBiome
 		surfaceLiquid = read.surfaceLiquid?.map({ Registries.liquid.ref(it) }, { Registries.liquid.ref(it) }) ?: BuiltinMetaMaterials.NO_LIQUID.ref

src/main/kotlin/ru/dbotthepony/kstarbound/defs/world/VisitableWorldParameters.kt

@@ -28,6 +28,7 @@ import ru.dbotthepony.kstarbound.math.vector.Vector2i
 import ru.dbotthepony.kstarbound.Starbound
 import ru.dbotthepony.kstarbound.collect.WeightedList
 import ru.dbotthepony.kstarbound.fromJson
+import ru.dbotthepony.kstarbound.fromJsonFast
 import ru.dbotthepony.kstarbound.io.readDouble
 import ru.dbotthepony.kstarbound.io.readInternedString
 import ru.dbotthepony.kstarbound.io.readNullable
@@ -36,6 +37,7 @@ import ru.dbotthepony.kstarbound.io.writeNullable
 import ru.dbotthepony.kstarbound.json.builder.DispatchingAdapter
 import ru.dbotthepony.kstarbound.json.builder.IStringSerializable
 import ru.dbotthepony.kstarbound.json.builder.JsonFactory
+import ru.dbotthepony.kstarbound.json.popObject
 import ru.dbotthepony.kstarbound.json.readJsonElement
 import ru.dbotthepony.kstarbound.json.readJsonObject
 import ru.dbotthepony.kstarbound.json.writeJsonObject
@@ -85,7 +87,7 @@ enum class VisitableWorldParametersType(override val jsonName: String, val token
 							return null
 
 						val instance = type.rawType.getDeclaredConstructor().newInstance() as VisitableWorldParameters
-						instance.fromJson(Starbound.ELEMENTS_ADAPTER.objects.read(`in`))
+						instance.fromJson(`in`.popObject())
 						return instance
 					}
 				} as TypeAdapter<T>
@@ -136,7 +138,7 @@ abstract class VisitableWorldParameters {
 		val threatLevel: Double,
 		val typeName: String,
 		val worldSize: Vector2i,
-		val gravity: Either<Double, Vector2d>,
+		val gravity: Either<Vector2d, Double>,
 		val airless: Boolean,
 		val environmentStatusEffects: Set<String>,
 		val overrideTech: Set<String>? = null,
@@ -149,12 +151,12 @@ abstract class VisitableWorldParameters {
 	)
 
 	open fun fromJson(data: JsonObject) {
-		val read = Starbound.gson.fromJson(data, StoreData::class.java)
+		val read = Starbound.gson.fromJsonFast(data, StoreData::class.java)
 
 		this.threatLevel = read.threatLevel
 		this.typeName = read.typeName
 		this.worldSize = read.worldSize
-		this.gravity = read.gravity.map({ Vector2d(y = it) }, { it })
+		this.gravity = read.gravity.map({ it }, { Vector2d(y = it) })
 		this.airless = read.airless
 		this.environmentStatusEffects = read.environmentStatusEffects
 		this.overrideTech = read.overrideTech
@@ -171,7 +173,7 @@ abstract class VisitableWorldParameters {
 			threatLevel,
 			typeName,
 			worldSize,
-			if (isLegacy) Either.left(gravity.y) else Either.right(gravity),
+			if (isLegacy) Either.right(gravity.y) else Either.left(gravity),
 			airless,
 			environmentStatusEffects,
 			overrideTech,

src/main/kotlin/ru/dbotthepony/kstarbound/json/BinaryJsonReader.kt

@@ -26,16 +26,22 @@ import java.io.InputStream
 import java.io.Reader
 import java.util.LinkedList
 import java.util.zip.DeflaterOutputStream
+import java.util.zip.Inflater
 import java.util.zip.InflaterInputStream
 
 private fun <T> ByteArray.callRead(inflate: Boolean, callable: DataInputStream.() -> T): T {
 	val stream = FastByteArrayInputStream(this)
 
 	if (inflate) {
-		val data = DataInputStream(BufferedInputStream(InflaterInputStream(stream)))
+		val inflater = Inflater()
-		val t = callable(data)
+
-		data.close()
+		try {
-		return t
+			val data = DataInputStream(BufferedInputStream(InflaterInputStream(stream, inflater, 0x4000), 0x10000))
+			val t = callable(data)
+			return t
+		} finally {
+			inflater.end()
+		}
 	} else {
 		return callable(DataInputStream(stream))
 	}

src/main/kotlin/ru/dbotthepony/kstarbound/json/BinaryJsonWriter.kt

@@ -5,24 +5,26 @@ import com.google.gson.JsonElement
 import com.google.gson.JsonNull
 import com.google.gson.JsonObject
 import com.google.gson.JsonPrimitive
-import it.unimi.dsi.fastutil.io.FastByteArrayInputStream
 import it.unimi.dsi.fastutil.io.FastByteArrayOutputStream
 import ru.dbotthepony.kommons.io.writeBinaryString
 import ru.dbotthepony.kommons.io.writeSignedVarLong
 import ru.dbotthepony.kommons.io.writeVarInt
 import java.io.BufferedOutputStream
-import java.io.DataInputStream
 import java.io.DataOutputStream
+import java.util.zip.Deflater
 import java.util.zip.DeflaterOutputStream
-import kotlin.math.absoluteValue
 
 private fun <T> T.callWrite(deflate: Boolean, callable: DataOutputStream.(T) -> Unit): ByteArray {
 	val stream = FastByteArrayOutputStream()
 
 	if (deflate) {
-		val data = DataOutputStream(BufferedOutputStream(DeflaterOutputStream(stream)))
+		val deflater = Deflater()
-		callable(data, this)
+		val data = DataOutputStream(BufferedOutputStream(DeflaterOutputStream(stream, deflater, 0x4000), 0x10000))
-		data.close()
+
+		data.use {
+			callable(data, this)
+		}
+
 	} else {
 		callable(DataOutputStream(stream), this)
 	}

src/main/kotlin/ru/dbotthepony/kstarbound/json/ContextualizedTypeAdapter.kt

@@ -1,47 +0,0 @@
-package ru.dbotthepony.kstarbound.json
-
-import com.google.gson.Gson
-import com.google.gson.JsonElement
-import com.google.gson.TypeAdapter
-import com.google.gson.TypeAdapterFactory
-import com.google.gson.internal.bind.JsonTreeReader
-import com.google.gson.reflect.TypeToken
-import com.google.gson.stream.JsonReader
-import com.google.gson.stream.JsonWriter
-import java.io.StringReader
-
-interface ContextualizedTypeAdapter<T> {
-	fun write(writer: JsonWriter, value: T, gson: Gson)
-
-	fun read(reader: JsonReader, gson: Gson): T
-
-	fun read(value: String, gson: Gson): T {
-		return read(JsonReader(StringReader(value)), gson)
-	}
-
-	fun read(value: JsonElement, gson: Gson): T {
-		return read(JsonTreeReader(value), gson)
-	}
-}
-
-inline fun <reified T> ContextualizedTypeAdapter<T>.factory() = ContextualizedTypeAdapterWrapper(T::class.java, this)
-
-class ContextualizedTypeAdapterWrapper<T>(val type: Class<T>, val adapter: ContextualizedTypeAdapter<T>) : TypeAdapterFactory {
-	override fun <T : Any?> create(gson: Gson, type: TypeToken<T>): TypeAdapter<T>? {
-		if (this.type.isAssignableFrom(type.rawType)) {
-			return Adapter(gson) as TypeAdapter<T>
-		}
-
-		return null
-	}
-
-	private inner class Adapter(private val gson: Gson) : TypeAdapter<T>() {
-		override fun write(out: JsonWriter, value: T) {
-			return adapter.write(out, value, gson)
-		}
-
-		override fun read(`in`: JsonReader): T {
-			return adapter.read(`in`, gson)
-		}
-	}
-}

src/main/kotlin/ru/dbotthepony/kstarbound/json/FastJsonTreeReader.kt

@@ -0,0 +1,255 @@
+package ru.dbotthepony.kstarbound.json
+
+import com.google.gson.JsonArray
+import com.google.gson.JsonElement
+import com.google.gson.JsonNull
+import com.google.gson.JsonObject
+import com.google.gson.JsonPrimitive
+import com.google.gson.stream.JsonReader
+import com.google.gson.stream.JsonToken
+import java.io.Reader
+
+// JsonTreeReader which allows to retrieve next json element instead of "reading" it
+//  through JsonReader methods
+class FastJsonTreeReader(top: JsonElement) : JsonReader(unreadable) {
+	private var stack = arrayOfNulls<Any>(32)
+	private var pathNames = arrayOfNulls<String>(32)
+	private var pathIndices = IntArray(32)
+	private var stackSize = 0
+
+	private fun push(element: Any?) {
+		if (stackSize >= stack.size) {
+			stack = stack.copyOf(stack.size * 2)
+			pathNames = pathNames.copyOf(stack.size)
+			pathIndices = pathIndices.copyOf(stack.size)
+		}
+
+		stack[stackSize++] = element
+	}
+
+	private fun pop(): Any? {
+		val result = stack[--stackSize]
+		stack[stackSize] = null
+		return result
+	}
+
+	private fun popWithShift(): Any? {
+		val result = pop()
+		if (stackSize > 0) pathIndices[stackSize - 1]++
+		return result
+	}
+
+	private fun peekStack() = stack[stackSize - 1]
+
+	init {
+		push(top)
+	}
+
+	override fun beginArray() {
+		expect(JsonToken.BEGIN_ARRAY)
+		push((peekStack() as JsonArray).iterator())
+		pathIndices[stackSize - 1] = 0
+	}
+
+	override fun endArray() {
+		expect(JsonToken.END_ARRAY)
+		pop() // empty iterator
+		pop() // array
+
+		if (stackSize > 0) {
+			pathIndices[stackSize - 1]++
+		}
+	}
+
+	override fun beginObject() {
+		expect(JsonToken.BEGIN_OBJECT)
+		push((peekStack() as JsonObject).entrySet().iterator())
+	}
+
+	override fun endObject() {
+		expect(JsonToken.END_OBJECT)
+		pop() // empty iterator
+		pop() // object
+
+		if (stackSize > 0) {
+			pathIndices[stackSize - 1]++
+		}
+	}
+
+	fun popObject(): JsonObject {
+		expect(JsonToken.BEGIN_OBJECT)
+		return popWithShift() as JsonObject
+	}
+
+	fun popArray(): JsonArray {
+		expect(JsonToken.BEGIN_ARRAY)
+		return popWithShift() as JsonArray
+	}
+
+	fun popJsonElement(): JsonElement {
+		if (stackSize == 0)
+			throw IllegalStateException("Reader is empty")
+
+		if (peekStack() is JsonElement) {
+			return popWithShift() as JsonElement
+		}
+
+		throw IllegalStateException("Not a json element${locationString()}")
+	}
+
+	override fun nextName(): String {
+		expect(JsonToken.NAME)
+		val (key, value) = (peekStack() as Iterator<Map.Entry<String, JsonElement>>).next()
+		pathNames[stackSize - 1] = key
+		push(value)
+		return key
+	}
+
+	override fun nextString(): String {
+		val peek = peek()
+		check(peek === JsonToken.STRING || peek === JsonToken.NUMBER) { "Expected STRING but was $peek${locationString()}" }
+		return (popWithShift() as JsonPrimitive).asString
+	}
+
+	override fun nextBoolean(): Boolean {
+		expect(JsonToken.BOOLEAN)
+		return (popWithShift() as JsonPrimitive).asBoolean
+	}
+
+	override fun nextNull() {
+		expect(JsonToken.NULL)
+		popWithShift()
+	}
+
+	override fun nextDouble(): Double {
+		val peek = peek()
+		check(peek === JsonToken.STRING || peek === JsonToken.NUMBER) { "Expected NUMBER but was $peek${locationString()}" }
+
+		val result = (popWithShift() as JsonPrimitive).asDouble
+
+		if (!isLenient && (result.isNaN() || result.isInfinite())) {
+			throw NumberFormatException("JSON forbids NaN and infinities: $result")
+		}
+
+		return result
+	}
+
+	override fun nextLong(): Long {
+		val peek = peek()
+		check(peek === JsonToken.STRING || peek === JsonToken.NUMBER) { "Expected NUMBER but was $peek${locationString()}" }
+		return (popWithShift() as JsonPrimitive).asLong
+	}
+
+	override fun nextInt(): Int {
+		val peek = peek()
+		check(peek === JsonToken.STRING || peek === JsonToken.NUMBER) { "Expected NUMBER but was $peek${locationString()}" }
+		return (popWithShift() as JsonPrimitive).asInt
+	}
+
+	override fun skipValue() {
+		if (peek() === JsonToken.NAME) {
+			nextName()
+			pathNames[stackSize - 2] = "null"
+		} else {
+			pop()
+			if (stackSize > 0) pathNames[stackSize - 1] = "null"
+		}
+
+		if (stackSize > 0) pathIndices[stackSize - 1]++
+	}
+
+	override fun close() {
+		// do nothing
+	}
+
+	override fun hasNext(): Boolean {
+		val peek = peek()
+		return peek != JsonToken.END_OBJECT && peek != JsonToken.END_ARRAY
+	}
+
+	override fun peek(): JsonToken {
+		if (stackSize == 0)
+			return JsonToken.END_DOCUMENT
+
+		val peek = stack[stackSize - 1]
+
+		return when (peek) {
+			is JsonObject -> JsonToken.BEGIN_OBJECT
+			is JsonArray -> JsonToken.BEGIN_ARRAY
+			is JsonPrimitive -> {
+				if (peek.isString) {
+					return JsonToken.STRING
+				} else if (peek.isBoolean) {
+					return JsonToken.BOOLEAN
+				} else if (peek.isNumber) {
+					return JsonToken.NUMBER
+				} else {
+					throw RuntimeException("unreachable code")
+				}
+			}
+
+			JsonNull.INSTANCE -> JsonToken.NULL
+
+			is Iterator<*> -> {
+				val isObject = stack[stackSize - 2] is JsonObject
+
+				if (peek.hasNext()) {
+					if (isObject) {
+						return JsonToken.NAME
+					} else {
+						push(peek.next())
+						return peek()
+					}
+				} else {
+					if (isObject) JsonToken.END_OBJECT else JsonToken.END_ARRAY
+				}
+			}
+
+			else -> throw RuntimeException()
+		}
+	}
+
+	override fun getPath(): String {
+		val result = StringBuilder().append('$')
+		var i = 0
+
+		while (i < stack.size) {
+			if (stack[i] is JsonArray) {
+				if (++i < stack.size && stack[i] is Iterator<*>) {
+					result.append('[').append(pathIndices[i]).append(']')
+				}
+			} else if (stack[i] is JsonObject) {
+				if (++i < stack.size && stack[i] is Iterator<*>) {
+					result.append('.')
+					if (pathNames[i] != null) {
+						result.append(pathNames[i])
+					}
+				}
+			}
+
+			i++
+		}
+
+		return result.toString()
+	}
+
+	private fun locationString(): String {
+		return " at path $path"
+	}
+
+	private fun expect(token: JsonToken) {
+		check(peek() === token) { "Expected $token but was ${peek()}${locationString()}" }
+	}
+
+	companion object {
+		private val unreadable = object : Reader() {
+			override fun read(cbuf: CharArray, off: Int, len: Int): Int {
+				throw AssertionError()
+			}
+
+			override fun close() {
+				throw AssertionError()
+			}
+		}
+	}
+}

src/main/kotlin/ru/dbotthepony/kstarbound/json/InternedJsonElementAdapter.kt

@@ -20,6 +20,9 @@ class InternedJsonElementAdapter(val stringInterner: Interner<String>) : TypeAda
 	}
 
 	override fun read(`in`: JsonReader): JsonElement {
+		if (`in` is FastJsonTreeReader)
+			return `in`.popJsonElement()
+
 		return when (val p = `in`.peek()) {
 			JsonToken.STRING -> JsonPrimitive(stringInterner.intern(`in`.nextString()))
 			JsonToken.NUMBER -> JsonPrimitive(LazilyParsedNumber(stringInterner.intern(`in`.nextString())))
@@ -45,6 +48,9 @@ class InternedJsonElementAdapter(val stringInterner: Interner<String>) : TypeAda
 			if (`in`.consumeNull())
 				return null
 
+			if (`in` is FastJsonTreeReader)
+				return `in`.popObject()
+
 			val output = JsonObject()
 			`in`.beginObject()
 			while (`in`.hasNext()) { output.add(stringInterner.intern(`in`.nextName()), this@InternedJsonElementAdapter.read(`in`)) }
@@ -62,6 +68,9 @@ class InternedJsonElementAdapter(val stringInterner: Interner<String>) : TypeAda
 			if (`in`.consumeNull())
 				return null
 
+			if (`in` is FastJsonTreeReader)
+				return `in`.popArray()
+
 			val output = JsonArray()
 			`in`.beginArray()
 			while (`in`.hasNext()) { output.add(this@InternedJsonElementAdapter.read(`in`)) }

src/main/kotlin/ru/dbotthepony/kstarbound/json/JsonUtils.kt

@@ -137,3 +137,31 @@ fun JsonObject.putAll(other: JsonObject, copy: Boolean = false): JsonObject {
 
 	return this
 }
+
+fun JsonReader.popObject(): JsonObject {
+	if (this is FastJsonTreeReader) {
+		return popObject()
+	} else {
+		return Starbound.ELEMENTS_ADAPTER.objects.read(this)
+	}
+}
+
+fun JsonReader.popArray(): JsonArray {
+	if (this is FastJsonTreeReader) {
+		return popArray()
+	} else {
+		return Starbound.ELEMENTS_ADAPTER.arrays.read(this)
+	}
+}
+
+fun JsonReader.popJsonElement(): JsonElement {
+	if (this is FastJsonTreeReader) {
+		return popJsonElement()
+	} else {
+		return Starbound.ELEMENTS_ADAPTER.read(this)
+	}
+}
+
+fun <T> TypeAdapter<T>.fromJsonTreeFast(tree: JsonElement): T {
+	return read(FastJsonTreeReader(tree))
+}

src/main/kotlin/ru/dbotthepony/kstarbound/json/builder/DispatchingAdapter.kt

@@ -12,6 +12,8 @@ import com.google.gson.stream.JsonWriter
 import ru.dbotthepony.kommons.gson.consumeNull
 import ru.dbotthepony.kommons.gson.value
 import ru.dbotthepony.kstarbound.Starbound
+import ru.dbotthepony.kstarbound.json.fromJsonTreeFast
+import ru.dbotthepony.kstarbound.json.popObject
 
 inline fun <reified E : Any, reified C : Any> DispatchingAdapter(
 	key: String,
@@ -60,6 +62,7 @@ class DispatchingAdapter<TYPE : Any, ELEMENT : Any>(
 					is JsonObject -> {
 						for ((k, v) in result.entrySet()) {
 							out.name(k)
+							// TODO: FastJsonTreeWriter OR BinaryJsonWriter
 							out.value(v)
 						}
 					}
@@ -75,10 +78,10 @@ class DispatchingAdapter<TYPE : Any, ELEMENT : Any>(
 			if (`in`.consumeNull())
 				return null
 
-			val read = Starbound.ELEMENTS_ADAPTER.objects.read(`in`)
+			val read = `in`.popObject()
 			val type = typeAdapter.fromJsonTree(read[key] ?: throw JsonSyntaxException("Missing '$key'"))
 			val adapter = adapters[type] ?: throw JsonSyntaxException("Unknown type $type (${read[key]})")
-			return adapter.fromJsonTree(read)
+			return adapter.fromJsonTreeFast(read)
 		}
 	}
 }

src/main/kotlin/ru/dbotthepony/kstarbound/json/builder/FactoryAdapter.kt

@@ -2,6 +2,7 @@ package ru.dbotthepony.kstarbound.json.builder
 
 import com.github.benmanes.caffeine.cache.Interner
 import com.google.common.collect.ImmutableList
+import com.google.common.collect.ImmutableMap
 import com.google.gson.Gson
 import com.google.gson.JsonElement
 import com.google.gson.JsonNull
@@ -20,10 +21,13 @@ import it.unimi.dsi.fastutil.ints.IntArrayList
 import it.unimi.dsi.fastutil.objects.Object2ObjectArrayMap
 import it.unimi.dsi.fastutil.objects.ObjectArraySet
 import org.apache.logging.log4j.LogManager
+import ru.dbotthepony.kommons.collect.stream
 import ru.dbotthepony.kommons.gson.consumeNull
 import ru.dbotthepony.kommons.gson.value
 import ru.dbotthepony.kommons.util.Either
 import ru.dbotthepony.kstarbound.Starbound
+import ru.dbotthepony.kstarbound.json.FastJsonTreeReader
+import ru.dbotthepony.kstarbound.json.popJsonElement
 import java.lang.reflect.Constructor
 import java.util.*
 import java.util.function.Function
@@ -46,7 +50,6 @@ class FactoryAdapter<T : Any> private constructor(
 	val asJsonArray: Boolean,
 	val stringInterner: Interner<String>,
 	val logMisses: Boolean,
-	private val elements: TypeAdapter<JsonElement>
 ) : TypeAdapter<T>() {
 	private val name2index = Object2ObjectArrayMap<String, IntArrayList>()
 	private val loggedMisses = Collections.synchronizedSet(ObjectArraySet<String>())
@@ -69,6 +72,12 @@ class FactoryAdapter<T : Any> private constructor(
 		}
 	}
 
+	private val hasFlatTypes = types.any { it.isFlat }
+	private val flatTypes = types
+		.withIndex()
+		.filter { it.value.isFlat }
+		.associate { it.index to it.value }
+
 	/**
 	 * Обычный конструктор класса (без флагов "значения по умолчанию")
 	 */
@@ -97,6 +106,13 @@ class FactoryAdapter<T : Any> private constructor(
 		return@first false
 	} ?: throw NoSuchElementException("Unable to determine constructor for ${clazz.qualifiedName} matching (${types.joinToString(", ")})")
 
+	// read() is magma hot method, so it must execute as quickly as possible,
+	//  we need to provide fast lookups
+	private data class KParameter(val isOptional: Boolean, val isMarkedNullable: Boolean)
+
+	private val regularFactoryParameters =
+		regularFactory.parameters.map { KParameter(it.isOptional, it.type.isMarkedNullable) }
+
 	/**
 	 * Синтетический конструктор класса, который создаётся Kotlin'ном, для создания классов со значениями по умолчанию
 	 */
@@ -228,14 +244,14 @@ class FactoryAdapter<T : Any> private constructor(
 		} else {
 			var json: JsonObject by Delegates.notNull()
 
-			if (types.any { it.isFlat }) {
+			if (hasFlatTypes) {
-				val readMap = elements.read(reader)
+				val readMap = reader.popJsonElement()
 
 				if (readMap !is JsonObject)
 					throw JsonParseException("Expected JSON element to be a Map, ${readMap::class.qualifiedName} given")
 
 				json = readMap
-				reader = JsonTreeReader(readMap)
+				reader = FastJsonTreeReader(readMap)
 			}
 
 			reader.beginObject()
@@ -256,7 +272,7 @@ class FactoryAdapter<T : Any> private constructor(
 					if (fields.size == 1) {
 						localReader = Either.right(reader)
 					} else {
-						val readValue = elements.read(reader)
+						val readValue = reader.popJsonElement()
 						localReader = Either.left(readValue)
 					}
 
@@ -295,18 +311,16 @@ class FactoryAdapter<T : Any> private constructor(
 				}
 			}
 
-			for ((i, property) in types.withIndex()) {
+			for ((i, property) in flatTypes) {
-				if (property.isFlat) {
+				try {
-					try {
+					val read = property.adapter.read(FastJsonTreeReader(json))
-						val read = property.adapter.read(JsonTreeReader(json))
 
-						if (read != null) {
+					if (read != null) {
-							presentValues[i] = true
+						presentValues[i] = true
-							readValues[i] = read
+						readValues[i] = read
-						}
-					} catch(err: Throwable) {
-						throw JsonSyntaxException("Reading flat field \"${property.property.name}\" for ${clazz.qualifiedName}", err)
 					}
+				} catch(err: Throwable) {
+					throw JsonSyntaxException("Reading flat field \"${property.property.name}\" for ${clazz.qualifiedName}", err)
 				}
 			}
 		}
@@ -326,7 +340,7 @@ class FactoryAdapter<T : Any> private constructor(
 				if (readValues[i] == null) {
 					if (!tuple.isMarkedNullable) {
 						throw JsonSyntaxException("Field ${field.name} of ${clazz.qualifiedName} does not accept nulls")
-					} else if (!regularFactory.parameters[i].isOptional && !presentValues[i]) {
+					} else if (!regularFactoryParameters[i].isOptional && !presentValues[i]) {
 						throw JsonSyntaxException("Field ${field.name} of ${clazz.qualifiedName} must be defined")
 					}
 				}
@@ -341,9 +355,9 @@ class FactoryAdapter<T : Any> private constructor(
 			readValues = readValues.copyOf(readValues.size + argumentFlagCount)
 
 			for ((i, field) in types.withIndex()) {
-				val param = regularFactory.parameters[i]
+				val param = regularFactoryParameters[i]
 
-				if (readValues[i] == null && param.isOptional && !param.type.isMarkedNullable) {
+				if (readValues[i] == null && param.isOptional && !param.isMarkedNullable) {
 					// while this makes whole shit way more lenient, at least it avoids silly errors
 					// caused by quirks in original engine serialization process
 					presentValues[i] = false
@@ -371,13 +385,13 @@ class FactoryAdapter<T : Any> private constructor(
 
 			for ((i, field) in types.withIndex()) {
 				if (readValues[i] != null) continue
-				val param = regularFactory.parameters[i]
+				val param = regularFactoryParameters[i]
 
 				if (param.isOptional && (!presentValues[i] || i in syntheticPrimitives)) {
 					readValues[i] = syntheticPrimitives[i]
 				} else if (!param.isOptional) {
 					if (!presentValues[i]) throw JsonSyntaxException("Field ${field.name} of ${clazz.qualifiedName} is missing")
-					if (!param.type.isMarkedNullable) throw JsonSyntaxException("Field ${field.name} of ${clazz.qualifiedName} does not accept nulls")
+					if (!param.isMarkedNullable) throw JsonSyntaxException("Field ${field.name} of ${clazz.qualifiedName} does not accept nulls")
 				}
 			}
 
@@ -427,7 +441,6 @@ class FactoryAdapter<T : Any> private constructor(
 				stringInterner = stringInterner,
 				aliases = aliases,
 				logMisses = logMisses,
-				elements = Starbound.ELEMENTS_ADAPTER
 			)
 		}
 

src/main/kotlin/ru/dbotthepony/kstarbound/network/packets/EntityMessagePacket.kt

@@ -45,23 +45,23 @@ class EntityMessagePacket(val entity: Either<Int, String>, val message: String,
 	}
 
 	private fun handle(connection: Connection, world: World<*, *>) {
-		if (entity.isLeft) {
+		val entity = if (entity.isLeft) {
-			val entity = world.entities[entity.left()]
+			world.entities[entity.left()]
-
-			if (entity == null) {
-				connection.send(EntityMessageResponsePacket(Either.left("No such entity ${this@EntityMessagePacket.entity}"), id))
-			} else {
-				entity.dispatchMessage(connection.connectionID, message, arguments)
-					.thenAccept(Consumer {
-						connection.send(EntityMessageResponsePacket(Either.right(it), id))
-					})
-					.exceptionally(Function {
-						connection.send(EntityMessageResponsePacket(Either.left(it.message ?: "Internal server error"), id))
-						null
-					})
-			}
 		} else {
-			TODO("messages to unique entities")
+			world.entities.values.firstOrNull { it.uniqueID.get() == entity.right() }
+		}
+
+		if (entity == null) {
+			connection.send(EntityMessageResponsePacket(Either.left("No such entity ${this@EntityMessagePacket.entity}"), id))
+		} else {
+			entity.dispatchMessage(connection.connectionID, message, arguments)
+				.thenAccept(Consumer {
+					connection.send(EntityMessageResponsePacket(Either.right(it), id))
+				})
+				.exceptionally(Function {
+					connection.send(EntityMessageResponsePacket(Either.left(it.message ?: "Internal server error"), id))
+					null
+				})
 		}
 	}
 

src/main/kotlin/ru/dbotthepony/kstarbound/server/ServerConnection.kt

@@ -5,6 +5,7 @@ import io.netty.channel.ChannelHandlerContext
 import it.unimi.dsi.fastutil.objects.ObjectOpenHashSet
 import kotlinx.coroutines.Job
 import kotlinx.coroutines.channels.Channel
+import kotlinx.coroutines.coroutineScope
 import kotlinx.coroutines.delay
 import kotlinx.coroutines.future.await
 import kotlinx.coroutines.launch
@@ -38,6 +39,7 @@ import ru.dbotthepony.kstarbound.server.world.WorldStorage
 import ru.dbotthepony.kstarbound.server.world.LegacyWorldStorage
 import ru.dbotthepony.kstarbound.server.world.ServerSystemWorld
 import ru.dbotthepony.kstarbound.server.world.ServerWorld
+import ru.dbotthepony.kstarbound.util.ActionPacer
 import ru.dbotthepony.kstarbound.world.SystemWorldLocation
 import ru.dbotthepony.kstarbound.world.UniversePos
 import java.util.UUID
@@ -399,6 +401,8 @@ class ServerConnection(val server: StarboundServer, type: ConnectionType) : Conn
 		}
 	}
 
+	private val celestialRequestsBudget = ActionPacer(64, 512)
+
 	private suspend fun handleCelestialRequests(requests: Collection<Either<Vector2i, Vector3i>>) {
 		val responses = ArrayList<Either<CelestialResponsePacket.ChunkData, CelestialResponsePacket.SystemData>>()
 
@@ -411,21 +415,26 @@ class ServerConnection(val server: StarboundServer, type: ConnectionType) : Conn
 				val systemParameters = HashMap<Vector3i, CelestialParameters>()
 				val planets = HashMap<Vector3i, HashMap<Int, CelestialResponsePacket.PlanetData>>()
 
-				for (system in systems) {
+				coroutineScope {
-					systemParameters[system.location] = server.universe.parameters(system) ?: continue
+					for (system in systems) {
+						launch {
+							celestialRequestsBudget.consume()
+							systemParameters[system.location] = server.universe.parameters(system) ?: return@launch
 
-					val systemPlanets = HashMap<Int, CelestialResponsePacket.PlanetData>()
+							val systemPlanets = HashMap<Int, CelestialResponsePacket.PlanetData>()
-					planets[system.location] = systemPlanets
+							planets[system.location] = systemPlanets
 
-					for (planetPos in server.universe.children(system)) {
+							for (planetPos in server.universe.children(system)) {
-						val parameters = server.universe.parameters(planetPos) ?: continue
+								val parameters = server.universe.parameters(planetPos) ?: continue
-						val satelliteMap = HashMap<Int, CelestialParameters>()
+								val satelliteMap = HashMap<Int, CelestialParameters>()
 
-						for (satellitePos in server.universe.children(planetPos)) {
+								for (satellitePos in server.universe.children(planetPos)) {
-							satelliteMap[satellitePos.satelliteOrbit] = server.universe.parameters(satellitePos) ?: continue
+									satelliteMap[satellitePos.satelliteOrbit] = server.universe.parameters(satellitePos) ?: continue
+								}
+
+								systemPlanets[planetPos.planetOrbit] = CelestialResponsePacket.PlanetData(parameters, satelliteMap)
+							}
 						}
-
-						systemPlanets[planetPos.planetOrbit] = CelestialResponsePacket.PlanetData(parameters, satelliteMap)
 					}
 				}
 
@@ -433,6 +442,9 @@ class ServerConnection(val server: StarboundServer, type: ConnectionType) : Conn
 					chunkPos, constellations, systemParameters, planets
 				)))
 			} else {
+				// even if malicious actor will do spread-out requests, which will generate new chunk each time
+				// we still handle requests sequentially
+				celestialRequestsBudget.consume()
 				val systemPos = UniversePos(request.right())
 				val map = HashMap<Int, CelestialResponsePacket.PlanetData>()
 

src/main/kotlin/ru/dbotthepony/kstarbound/server/world/ServerUniverse.kt

@@ -42,6 +42,7 @@ import ru.dbotthepony.kstarbound.util.random.AbstractPerlinNoise
 import ru.dbotthepony.kstarbound.util.random.nextRange
 import ru.dbotthepony.kstarbound.util.random.random
 import ru.dbotthepony.kstarbound.util.random.staticRandom64
+import ru.dbotthepony.kstarbound.util.supplyAsync
 import ru.dbotthepony.kstarbound.world.Universe
 import ru.dbotthepony.kstarbound.world.UniversePos
 import java.io.Closeable
@@ -181,15 +182,6 @@ class ServerUniverse(folder: File? = null) : Universe(), Closeable {
 	}
 
 	private data class System(val x: Int, val y: Int, val z: Int, val parameters: CelestialParameters, val planets: Map<Pair<Int, Int>, CelestialParameters>) {
-		constructor(x: Int, y: Int, z: Int, data: ResultSet) : this(
-			x, y, z,
-			Starbound.gson.fromJson(data.getBytes(1).readJsonElementInflated())!!,
-			data.getBytes(2).readJsonArrayInflated().associate {
-				it as JsonArray
-				(it[0].asInt to it[1].asInt) to Starbound.gson.fromJson(it[2])!!
-			}
-		)
-
 		fun parameters(pos: UniversePos): CelestialParameters? {
 			if (pos.isSystem) {
 				return parameters
@@ -266,14 +258,27 @@ class ServerUniverse(folder: File? = null) : Universe(), Closeable {
 		.executor(Starbound.EXECUTOR)
 		.build<Vector3i, CompletableFuture<System?>>()
 
-	private fun loadSystem(pos: Vector3i): System? {
+	private fun loadSystem(pos: Vector3i): CompletableFuture<System>? {
 		selectSystem.setInt(1, pos.x)
 		selectSystem.setInt(2, pos.y)
 		selectSystem.setInt(3, pos.z)
 
 		return selectSystem.executeQuery().use {
 			if (it.next()) {
-				System(pos.x, pos.y, pos.z, it)
+				val parametersBytes = it.getBytes(1)
+				val planetsBytes = it.getBytes(2)
+
+				// deserialize in off-thread since it involves big json structures
+				Starbound.EXECUTOR.supplyAsync {
+					val parameters: CelestialParameters = Starbound.gson.fromJson(parametersBytes.readJsonElementInflated())!!
+
+					val planets: Map<Pair<Int, Int>, CelestialParameters> = planetsBytes.readJsonArrayInflated().associate {
+						it as JsonArray
+						(it[0].asInt to it[1].asInt) to Starbound.gson.fromJson(it[2])!!
+					}
+
+					System(pos.x, pos.y, pos.z, parameters, planets)
+				}
 			} else {
 				null
 			}
@@ -285,8 +290,9 @@ class ServerUniverse(folder: File? = null) : Universe(), Closeable {
 
 		if (existing != null) {
 			// hit, system already exists
+			val wait = existing.await()
 			systemFutures.remove(pos)
-			return existing
+			return wait
 		}
 
 		// lets try to get chunk this system is in
@@ -298,7 +304,7 @@ class ServerUniverse(folder: File? = null) : Universe(), Closeable {
 		if (pos !in chunk.systems)
 			return null
 
-		return loadSystem(pos)
+		return loadSystem(pos)!!.await()
 	}
 
 	private fun getSystem(pos: Vector3i): CompletableFuture<System?> {