diff --git a/src/main/kotlin/ru/dbotthepony/kstarbound/Ext.kt b/src/main/kotlin/ru/dbotthepony/kstarbound/Ext.kt index f6c59cef..160fbc05 100644 --- a/src/main/kotlin/ru/dbotthepony/kstarbound/Ext.kt +++ b/src/main/kotlin/ru/dbotthepony/kstarbound/Ext.kt @@ -9,6 +9,7 @@ import com.google.gson.TypeAdapterFactory import com.google.gson.reflect.TypeToken import com.google.gson.stream.JsonReader import ru.dbotthepony.kommons.util.KOptional +import ru.dbotthepony.kstarbound.json.FastJsonTreeReader import java.util.Arrays import java.util.concurrent.Callable import java.util.concurrent.ForkJoinPool @@ -49,7 +50,9 @@ operator fun ImmutableMap.Builder.set(key: K, value: V) fun String.sintern(): String = Starbound.STRINGS.intern(this) inline fun Gson.fromJson(reader: JsonReader): T? = fromJson(reader, T::class.java) -inline fun Gson.fromJson(reader: JsonElement): T? = fromJson(reader, T::class.java) +inline fun Gson.fromJson(reader: JsonElement): T? = getAdapter(T::class.java).read(FastJsonTreeReader(reader)) + +fun Gson.fromJsonFast(reader: JsonElement, type: Class): T = getAdapter(type).read(FastJsonTreeReader(reader)) /** * guarantees even distribution of tasks while also preserving encountered order of elements diff --git a/src/main/kotlin/ru/dbotthepony/kstarbound/defs/world/TerrestrialWorldParameters.kt b/src/main/kotlin/ru/dbotthepony/kstarbound/defs/world/TerrestrialWorldParameters.kt index 06d75530..d927dd21 100644 --- a/src/main/kotlin/ru/dbotthepony/kstarbound/defs/world/TerrestrialWorldParameters.kt +++ b/src/main/kotlin/ru/dbotthepony/kstarbound/defs/world/TerrestrialWorldParameters.kt @@ -30,6 +30,7 @@ import ru.dbotthepony.kstarbound.defs.tile.BuiltinMetaMaterials import ru.dbotthepony.kstarbound.defs.tile.LiquidDefinition import ru.dbotthepony.kstarbound.defs.tile.isEmptyLiquid import ru.dbotthepony.kstarbound.fromJson +import ru.dbotthepony.kstarbound.fromJsonFast import ru.dbotthepony.kstarbound.io.readInternedString import ru.dbotthepony.kstarbound.io.readVector2d import ru.dbotthepony.kstarbound.io.writeStruct2d @@ -182,7 +183,7 @@ class TerrestrialWorldParameters : VisitableWorldParameters() { override fun fromJson(data: JsonObject) { super.fromJson(data) - val read = Starbound.gson.fromJson(data, StoreData::class.java) + val read = Starbound.gson.fromJsonFast(data, StoreData::class.java) primaryBiome = read.primaryBiome surfaceLiquid = read.surfaceLiquid?.map({ Registries.liquid.ref(it) }, { Registries.liquid.ref(it) }) ?: BuiltinMetaMaterials.NO_LIQUID.ref diff --git a/src/main/kotlin/ru/dbotthepony/kstarbound/defs/world/VisitableWorldParameters.kt b/src/main/kotlin/ru/dbotthepony/kstarbound/defs/world/VisitableWorldParameters.kt index 8e7d5423..c3373198 100644 --- a/src/main/kotlin/ru/dbotthepony/kstarbound/defs/world/VisitableWorldParameters.kt +++ b/src/main/kotlin/ru/dbotthepony/kstarbound/defs/world/VisitableWorldParameters.kt @@ -28,6 +28,7 @@ import ru.dbotthepony.kstarbound.math.vector.Vector2i import ru.dbotthepony.kstarbound.Starbound import ru.dbotthepony.kstarbound.collect.WeightedList import ru.dbotthepony.kstarbound.fromJson +import ru.dbotthepony.kstarbound.fromJsonFast import ru.dbotthepony.kstarbound.io.readDouble import ru.dbotthepony.kstarbound.io.readInternedString import ru.dbotthepony.kstarbound.io.readNullable @@ -36,6 +37,7 @@ import ru.dbotthepony.kstarbound.io.writeNullable import ru.dbotthepony.kstarbound.json.builder.DispatchingAdapter import ru.dbotthepony.kstarbound.json.builder.IStringSerializable import ru.dbotthepony.kstarbound.json.builder.JsonFactory +import ru.dbotthepony.kstarbound.json.popObject import ru.dbotthepony.kstarbound.json.readJsonElement import ru.dbotthepony.kstarbound.json.readJsonObject import ru.dbotthepony.kstarbound.json.writeJsonObject @@ -85,7 +87,7 @@ enum class VisitableWorldParametersType(override val jsonName: String, val token return null val instance = type.rawType.getDeclaredConstructor().newInstance() as VisitableWorldParameters - instance.fromJson(Starbound.ELEMENTS_ADAPTER.objects.read(`in`)) + instance.fromJson(`in`.popObject()) return instance } } as TypeAdapter @@ -136,7 +138,7 @@ abstract class VisitableWorldParameters { val threatLevel: Double, val typeName: String, val worldSize: Vector2i, - val gravity: Either, + val gravity: Either, val airless: Boolean, val environmentStatusEffects: Set, val overrideTech: Set? = null, @@ -149,12 +151,12 @@ abstract class VisitableWorldParameters { ) open fun fromJson(data: JsonObject) { - val read = Starbound.gson.fromJson(data, StoreData::class.java) + val read = Starbound.gson.fromJsonFast(data, StoreData::class.java) this.threatLevel = read.threatLevel this.typeName = read.typeName this.worldSize = read.worldSize - this.gravity = read.gravity.map({ Vector2d(y = it) }, { it }) + this.gravity = read.gravity.map({ it }, { Vector2d(y = it) }) this.airless = read.airless this.environmentStatusEffects = read.environmentStatusEffects this.overrideTech = read.overrideTech @@ -171,7 +173,7 @@ abstract class VisitableWorldParameters { threatLevel, typeName, worldSize, - if (isLegacy) Either.left(gravity.y) else Either.right(gravity), + if (isLegacy) Either.right(gravity.y) else Either.left(gravity), airless, environmentStatusEffects, overrideTech, diff --git a/src/main/kotlin/ru/dbotthepony/kstarbound/json/BinaryJsonReader.kt b/src/main/kotlin/ru/dbotthepony/kstarbound/json/BinaryJsonReader.kt index 59226fcc..af66ad38 100644 --- a/src/main/kotlin/ru/dbotthepony/kstarbound/json/BinaryJsonReader.kt +++ b/src/main/kotlin/ru/dbotthepony/kstarbound/json/BinaryJsonReader.kt @@ -26,16 +26,22 @@ import java.io.InputStream import java.io.Reader import java.util.LinkedList import java.util.zip.DeflaterOutputStream +import java.util.zip.Inflater import java.util.zip.InflaterInputStream private fun ByteArray.callRead(inflate: Boolean, callable: DataInputStream.() -> T): T { val stream = FastByteArrayInputStream(this) if (inflate) { - val data = DataInputStream(BufferedInputStream(InflaterInputStream(stream))) - val t = callable(data) - data.close() - return t + val inflater = Inflater() + + try { + val data = DataInputStream(BufferedInputStream(InflaterInputStream(stream, inflater, 0x4000), 0x10000)) + val t = callable(data) + return t + } finally { + inflater.end() + } } else { return callable(DataInputStream(stream)) } diff --git a/src/main/kotlin/ru/dbotthepony/kstarbound/json/BinaryJsonWriter.kt b/src/main/kotlin/ru/dbotthepony/kstarbound/json/BinaryJsonWriter.kt index 4ae059df..ae689ed4 100644 --- a/src/main/kotlin/ru/dbotthepony/kstarbound/json/BinaryJsonWriter.kt +++ b/src/main/kotlin/ru/dbotthepony/kstarbound/json/BinaryJsonWriter.kt @@ -5,24 +5,26 @@ import com.google.gson.JsonElement import com.google.gson.JsonNull import com.google.gson.JsonObject import com.google.gson.JsonPrimitive -import it.unimi.dsi.fastutil.io.FastByteArrayInputStream import it.unimi.dsi.fastutil.io.FastByteArrayOutputStream import ru.dbotthepony.kommons.io.writeBinaryString import ru.dbotthepony.kommons.io.writeSignedVarLong import ru.dbotthepony.kommons.io.writeVarInt import java.io.BufferedOutputStream -import java.io.DataInputStream import java.io.DataOutputStream +import java.util.zip.Deflater import java.util.zip.DeflaterOutputStream -import kotlin.math.absoluteValue private fun T.callWrite(deflate: Boolean, callable: DataOutputStream.(T) -> Unit): ByteArray { val stream = FastByteArrayOutputStream() if (deflate) { - val data = DataOutputStream(BufferedOutputStream(DeflaterOutputStream(stream))) - callable(data, this) - data.close() + val deflater = Deflater() + val data = DataOutputStream(BufferedOutputStream(DeflaterOutputStream(stream, deflater, 0x4000), 0x10000)) + + data.use { + callable(data, this) + } + } else { callable(DataOutputStream(stream), this) } diff --git a/src/main/kotlin/ru/dbotthepony/kstarbound/json/ContextualizedTypeAdapter.kt b/src/main/kotlin/ru/dbotthepony/kstarbound/json/ContextualizedTypeAdapter.kt deleted file mode 100644 index 1a523715..00000000 --- a/src/main/kotlin/ru/dbotthepony/kstarbound/json/ContextualizedTypeAdapter.kt +++ /dev/null @@ -1,47 +0,0 @@ -package ru.dbotthepony.kstarbound.json - -import com.google.gson.Gson -import com.google.gson.JsonElement -import com.google.gson.TypeAdapter -import com.google.gson.TypeAdapterFactory -import com.google.gson.internal.bind.JsonTreeReader -import com.google.gson.reflect.TypeToken -import com.google.gson.stream.JsonReader -import com.google.gson.stream.JsonWriter -import java.io.StringReader - -interface ContextualizedTypeAdapter { - fun write(writer: JsonWriter, value: T, gson: Gson) - - fun read(reader: JsonReader, gson: Gson): T - - fun read(value: String, gson: Gson): T { - return read(JsonReader(StringReader(value)), gson) - } - - fun read(value: JsonElement, gson: Gson): T { - return read(JsonTreeReader(value), gson) - } -} - -inline fun ContextualizedTypeAdapter.factory() = ContextualizedTypeAdapterWrapper(T::class.java, this) - -class ContextualizedTypeAdapterWrapper(val type: Class, val adapter: ContextualizedTypeAdapter) : TypeAdapterFactory { - override fun create(gson: Gson, type: TypeToken): TypeAdapter? { - if (this.type.isAssignableFrom(type.rawType)) { - return Adapter(gson) as TypeAdapter - } - - return null - } - - private inner class Adapter(private val gson: Gson) : TypeAdapter() { - override fun write(out: JsonWriter, value: T) { - return adapter.write(out, value, gson) - } - - override fun read(`in`: JsonReader): T { - return adapter.read(`in`, gson) - } - } -} diff --git a/src/main/kotlin/ru/dbotthepony/kstarbound/json/FastJsonTreeReader.kt b/src/main/kotlin/ru/dbotthepony/kstarbound/json/FastJsonTreeReader.kt new file mode 100644 index 00000000..5daf5999 --- /dev/null +++ b/src/main/kotlin/ru/dbotthepony/kstarbound/json/FastJsonTreeReader.kt @@ -0,0 +1,255 @@ +package ru.dbotthepony.kstarbound.json + +import com.google.gson.JsonArray +import com.google.gson.JsonElement +import com.google.gson.JsonNull +import com.google.gson.JsonObject +import com.google.gson.JsonPrimitive +import com.google.gson.stream.JsonReader +import com.google.gson.stream.JsonToken +import java.io.Reader + +// JsonTreeReader which allows to retrieve next json element instead of "reading" it +// through JsonReader methods +class FastJsonTreeReader(top: JsonElement) : JsonReader(unreadable) { + private var stack = arrayOfNulls(32) + private var pathNames = arrayOfNulls(32) + private var pathIndices = IntArray(32) + private var stackSize = 0 + + private fun push(element: Any?) { + if (stackSize >= stack.size) { + stack = stack.copyOf(stack.size * 2) + pathNames = pathNames.copyOf(stack.size) + pathIndices = pathIndices.copyOf(stack.size) + } + + stack[stackSize++] = element + } + + private fun pop(): Any? { + val result = stack[--stackSize] + stack[stackSize] = null + return result + } + + private fun popWithShift(): Any? { + val result = pop() + if (stackSize > 0) pathIndices[stackSize - 1]++ + return result + } + + private fun peekStack() = stack[stackSize - 1] + + init { + push(top) + } + + override fun beginArray() { + expect(JsonToken.BEGIN_ARRAY) + push((peekStack() as JsonArray).iterator()) + pathIndices[stackSize - 1] = 0 + } + + override fun endArray() { + expect(JsonToken.END_ARRAY) + pop() // empty iterator + pop() // array + + if (stackSize > 0) { + pathIndices[stackSize - 1]++ + } + } + + override fun beginObject() { + expect(JsonToken.BEGIN_OBJECT) + push((peekStack() as JsonObject).entrySet().iterator()) + } + + override fun endObject() { + expect(JsonToken.END_OBJECT) + pop() // empty iterator + pop() // object + + if (stackSize > 0) { + pathIndices[stackSize - 1]++ + } + } + + fun popObject(): JsonObject { + expect(JsonToken.BEGIN_OBJECT) + return popWithShift() as JsonObject + } + + fun popArray(): JsonArray { + expect(JsonToken.BEGIN_ARRAY) + return popWithShift() as JsonArray + } + + fun popJsonElement(): JsonElement { + if (stackSize == 0) + throw IllegalStateException("Reader is empty") + + if (peekStack() is JsonElement) { + return popWithShift() as JsonElement + } + + throw IllegalStateException("Not a json element${locationString()}") + } + + override fun nextName(): String { + expect(JsonToken.NAME) + val (key, value) = (peekStack() as Iterator>).next() + pathNames[stackSize - 1] = key + push(value) + return key + } + + override fun nextString(): String { + val peek = peek() + check(peek === JsonToken.STRING || peek === JsonToken.NUMBER) { "Expected STRING but was $peek${locationString()}" } + return (popWithShift() as JsonPrimitive).asString + } + + override fun nextBoolean(): Boolean { + expect(JsonToken.BOOLEAN) + return (popWithShift() as JsonPrimitive).asBoolean + } + + override fun nextNull() { + expect(JsonToken.NULL) + popWithShift() + } + + override fun nextDouble(): Double { + val peek = peek() + check(peek === JsonToken.STRING || peek === JsonToken.NUMBER) { "Expected NUMBER but was $peek${locationString()}" } + + val result = (popWithShift() as JsonPrimitive).asDouble + + if (!isLenient && (result.isNaN() || result.isInfinite())) { + throw NumberFormatException("JSON forbids NaN and infinities: $result") + } + + return result + } + + override fun nextLong(): Long { + val peek = peek() + check(peek === JsonToken.STRING || peek === JsonToken.NUMBER) { "Expected NUMBER but was $peek${locationString()}" } + return (popWithShift() as JsonPrimitive).asLong + } + + override fun nextInt(): Int { + val peek = peek() + check(peek === JsonToken.STRING || peek === JsonToken.NUMBER) { "Expected NUMBER but was $peek${locationString()}" } + return (popWithShift() as JsonPrimitive).asInt + } + + override fun skipValue() { + if (peek() === JsonToken.NAME) { + nextName() + pathNames[stackSize - 2] = "null" + } else { + pop() + if (stackSize > 0) pathNames[stackSize - 1] = "null" + } + + if (stackSize > 0) pathIndices[stackSize - 1]++ + } + + override fun close() { + // do nothing + } + + override fun hasNext(): Boolean { + val peek = peek() + return peek != JsonToken.END_OBJECT && peek != JsonToken.END_ARRAY + } + + override fun peek(): JsonToken { + if (stackSize == 0) + return JsonToken.END_DOCUMENT + + val peek = stack[stackSize - 1] + + return when (peek) { + is JsonObject -> JsonToken.BEGIN_OBJECT + is JsonArray -> JsonToken.BEGIN_ARRAY + is JsonPrimitive -> { + if (peek.isString) { + return JsonToken.STRING + } else if (peek.isBoolean) { + return JsonToken.BOOLEAN + } else if (peek.isNumber) { + return JsonToken.NUMBER + } else { + throw RuntimeException("unreachable code") + } + } + + JsonNull.INSTANCE -> JsonToken.NULL + + is Iterator<*> -> { + val isObject = stack[stackSize - 2] is JsonObject + + if (peek.hasNext()) { + if (isObject) { + return JsonToken.NAME + } else { + push(peek.next()) + return peek() + } + } else { + if (isObject) JsonToken.END_OBJECT else JsonToken.END_ARRAY + } + } + + else -> throw RuntimeException() + } + } + + override fun getPath(): String { + val result = StringBuilder().append('$') + var i = 0 + + while (i < stack.size) { + if (stack[i] is JsonArray) { + if (++i < stack.size && stack[i] is Iterator<*>) { + result.append('[').append(pathIndices[i]).append(']') + } + } else if (stack[i] is JsonObject) { + if (++i < stack.size && stack[i] is Iterator<*>) { + result.append('.') + if (pathNames[i] != null) { + result.append(pathNames[i]) + } + } + } + + i++ + } + + return result.toString() + } + + private fun locationString(): String { + return " at path $path" + } + + private fun expect(token: JsonToken) { + check(peek() === token) { "Expected $token but was ${peek()}${locationString()}" } + } + + companion object { + private val unreadable = object : Reader() { + override fun read(cbuf: CharArray, off: Int, len: Int): Int { + throw AssertionError() + } + + override fun close() { + throw AssertionError() + } + } + } +} diff --git a/src/main/kotlin/ru/dbotthepony/kstarbound/json/InternedJsonElementAdapter.kt b/src/main/kotlin/ru/dbotthepony/kstarbound/json/InternedJsonElementAdapter.kt index d75cbe11..fe87c8d8 100644 --- a/src/main/kotlin/ru/dbotthepony/kstarbound/json/InternedJsonElementAdapter.kt +++ b/src/main/kotlin/ru/dbotthepony/kstarbound/json/InternedJsonElementAdapter.kt @@ -20,6 +20,9 @@ class InternedJsonElementAdapter(val stringInterner: Interner) : TypeAda } override fun read(`in`: JsonReader): JsonElement { + if (`in` is FastJsonTreeReader) + return `in`.popJsonElement() + return when (val p = `in`.peek()) { JsonToken.STRING -> JsonPrimitive(stringInterner.intern(`in`.nextString())) JsonToken.NUMBER -> JsonPrimitive(LazilyParsedNumber(stringInterner.intern(`in`.nextString()))) @@ -45,6 +48,9 @@ class InternedJsonElementAdapter(val stringInterner: Interner) : TypeAda if (`in`.consumeNull()) return null + if (`in` is FastJsonTreeReader) + return `in`.popObject() + val output = JsonObject() `in`.beginObject() while (`in`.hasNext()) { output.add(stringInterner.intern(`in`.nextName()), this@InternedJsonElementAdapter.read(`in`)) } @@ -62,6 +68,9 @@ class InternedJsonElementAdapter(val stringInterner: Interner) : TypeAda if (`in`.consumeNull()) return null + if (`in` is FastJsonTreeReader) + return `in`.popArray() + val output = JsonArray() `in`.beginArray() while (`in`.hasNext()) { output.add(this@InternedJsonElementAdapter.read(`in`)) } diff --git a/src/main/kotlin/ru/dbotthepony/kstarbound/json/JsonUtils.kt b/src/main/kotlin/ru/dbotthepony/kstarbound/json/JsonUtils.kt index 5f2cef42..fa6b05b7 100644 --- a/src/main/kotlin/ru/dbotthepony/kstarbound/json/JsonUtils.kt +++ b/src/main/kotlin/ru/dbotthepony/kstarbound/json/JsonUtils.kt @@ -137,3 +137,31 @@ fun JsonObject.putAll(other: JsonObject, copy: Boolean = false): JsonObject { return this } + +fun JsonReader.popObject(): JsonObject { + if (this is FastJsonTreeReader) { + return popObject() + } else { + return Starbound.ELEMENTS_ADAPTER.objects.read(this) + } +} + +fun JsonReader.popArray(): JsonArray { + if (this is FastJsonTreeReader) { + return popArray() + } else { + return Starbound.ELEMENTS_ADAPTER.arrays.read(this) + } +} + +fun JsonReader.popJsonElement(): JsonElement { + if (this is FastJsonTreeReader) { + return popJsonElement() + } else { + return Starbound.ELEMENTS_ADAPTER.read(this) + } +} + +fun TypeAdapter.fromJsonTreeFast(tree: JsonElement): T { + return read(FastJsonTreeReader(tree)) +} diff --git a/src/main/kotlin/ru/dbotthepony/kstarbound/json/builder/DispatchingAdapter.kt b/src/main/kotlin/ru/dbotthepony/kstarbound/json/builder/DispatchingAdapter.kt index 3adfcd3f..46f8af2a 100644 --- a/src/main/kotlin/ru/dbotthepony/kstarbound/json/builder/DispatchingAdapter.kt +++ b/src/main/kotlin/ru/dbotthepony/kstarbound/json/builder/DispatchingAdapter.kt @@ -12,6 +12,8 @@ import com.google.gson.stream.JsonWriter import ru.dbotthepony.kommons.gson.consumeNull import ru.dbotthepony.kommons.gson.value import ru.dbotthepony.kstarbound.Starbound +import ru.dbotthepony.kstarbound.json.fromJsonTreeFast +import ru.dbotthepony.kstarbound.json.popObject inline fun DispatchingAdapter( key: String, @@ -60,6 +62,7 @@ class DispatchingAdapter( is JsonObject -> { for ((k, v) in result.entrySet()) { out.name(k) + // TODO: FastJsonTreeWriter OR BinaryJsonWriter out.value(v) } } @@ -75,10 +78,10 @@ class DispatchingAdapter( if (`in`.consumeNull()) return null - val read = Starbound.ELEMENTS_ADAPTER.objects.read(`in`) + val read = `in`.popObject() val type = typeAdapter.fromJsonTree(read[key] ?: throw JsonSyntaxException("Missing '$key'")) val adapter = adapters[type] ?: throw JsonSyntaxException("Unknown type $type (${read[key]})") - return adapter.fromJsonTree(read) + return adapter.fromJsonTreeFast(read) } } } diff --git a/src/main/kotlin/ru/dbotthepony/kstarbound/json/builder/FactoryAdapter.kt b/src/main/kotlin/ru/dbotthepony/kstarbound/json/builder/FactoryAdapter.kt index c77ff02a..4567fa21 100644 --- a/src/main/kotlin/ru/dbotthepony/kstarbound/json/builder/FactoryAdapter.kt +++ b/src/main/kotlin/ru/dbotthepony/kstarbound/json/builder/FactoryAdapter.kt @@ -2,6 +2,7 @@ package ru.dbotthepony.kstarbound.json.builder import com.github.benmanes.caffeine.cache.Interner import com.google.common.collect.ImmutableList +import com.google.common.collect.ImmutableMap import com.google.gson.Gson import com.google.gson.JsonElement import com.google.gson.JsonNull @@ -20,10 +21,13 @@ import it.unimi.dsi.fastutil.ints.IntArrayList import it.unimi.dsi.fastutil.objects.Object2ObjectArrayMap import it.unimi.dsi.fastutil.objects.ObjectArraySet import org.apache.logging.log4j.LogManager +import ru.dbotthepony.kommons.collect.stream import ru.dbotthepony.kommons.gson.consumeNull import ru.dbotthepony.kommons.gson.value import ru.dbotthepony.kommons.util.Either import ru.dbotthepony.kstarbound.Starbound +import ru.dbotthepony.kstarbound.json.FastJsonTreeReader +import ru.dbotthepony.kstarbound.json.popJsonElement import java.lang.reflect.Constructor import java.util.* import java.util.function.Function @@ -46,7 +50,6 @@ class FactoryAdapter private constructor( val asJsonArray: Boolean, val stringInterner: Interner, val logMisses: Boolean, - private val elements: TypeAdapter ) : TypeAdapter() { private val name2index = Object2ObjectArrayMap() private val loggedMisses = Collections.synchronizedSet(ObjectArraySet()) @@ -69,6 +72,12 @@ class FactoryAdapter private constructor( } } + private val hasFlatTypes = types.any { it.isFlat } + private val flatTypes = types + .withIndex() + .filter { it.value.isFlat } + .associate { it.index to it.value } + /** * Обычный конструктор класса (без флагов "значения по умолчанию") */ @@ -97,6 +106,13 @@ class FactoryAdapter private constructor( return@first false } ?: throw NoSuchElementException("Unable to determine constructor for ${clazz.qualifiedName} matching (${types.joinToString(", ")})") + // read() is magma hot method, so it must execute as quickly as possible, + // we need to provide fast lookups + private data class KParameter(val isOptional: Boolean, val isMarkedNullable: Boolean) + + private val regularFactoryParameters = + regularFactory.parameters.map { KParameter(it.isOptional, it.type.isMarkedNullable) } + /** * Синтетический конструктор класса, который создаётся Kotlin'ном, для создания классов со значениями по умолчанию */ @@ -228,14 +244,14 @@ class FactoryAdapter private constructor( } else { var json: JsonObject by Delegates.notNull() - if (types.any { it.isFlat }) { - val readMap = elements.read(reader) + if (hasFlatTypes) { + val readMap = reader.popJsonElement() if (readMap !is JsonObject) throw JsonParseException("Expected JSON element to be a Map, ${readMap::class.qualifiedName} given") json = readMap - reader = JsonTreeReader(readMap) + reader = FastJsonTreeReader(readMap) } reader.beginObject() @@ -256,7 +272,7 @@ class FactoryAdapter private constructor( if (fields.size == 1) { localReader = Either.right(reader) } else { - val readValue = elements.read(reader) + val readValue = reader.popJsonElement() localReader = Either.left(readValue) } @@ -295,18 +311,16 @@ class FactoryAdapter private constructor( } } - for ((i, property) in types.withIndex()) { - if (property.isFlat) { - try { - val read = property.adapter.read(JsonTreeReader(json)) + for ((i, property) in flatTypes) { + try { + val read = property.adapter.read(FastJsonTreeReader(json)) - if (read != null) { - presentValues[i] = true - readValues[i] = read - } - } catch(err: Throwable) { - throw JsonSyntaxException("Reading flat field \"${property.property.name}\" for ${clazz.qualifiedName}", err) + if (read != null) { + presentValues[i] = true + readValues[i] = read } + } catch(err: Throwable) { + throw JsonSyntaxException("Reading flat field \"${property.property.name}\" for ${clazz.qualifiedName}", err) } } } @@ -326,7 +340,7 @@ class FactoryAdapter private constructor( if (readValues[i] == null) { if (!tuple.isMarkedNullable) { throw JsonSyntaxException("Field ${field.name} of ${clazz.qualifiedName} does not accept nulls") - } else if (!regularFactory.parameters[i].isOptional && !presentValues[i]) { + } else if (!regularFactoryParameters[i].isOptional && !presentValues[i]) { throw JsonSyntaxException("Field ${field.name} of ${clazz.qualifiedName} must be defined") } } @@ -341,9 +355,9 @@ class FactoryAdapter private constructor( readValues = readValues.copyOf(readValues.size + argumentFlagCount) for ((i, field) in types.withIndex()) { - val param = regularFactory.parameters[i] + val param = regularFactoryParameters[i] - if (readValues[i] == null && param.isOptional && !param.type.isMarkedNullable) { + if (readValues[i] == null && param.isOptional && !param.isMarkedNullable) { // while this makes whole shit way more lenient, at least it avoids silly errors // caused by quirks in original engine serialization process presentValues[i] = false @@ -371,13 +385,13 @@ class FactoryAdapter private constructor( for ((i, field) in types.withIndex()) { if (readValues[i] != null) continue - val param = regularFactory.parameters[i] + val param = regularFactoryParameters[i] if (param.isOptional && (!presentValues[i] || i in syntheticPrimitives)) { readValues[i] = syntheticPrimitives[i] } else if (!param.isOptional) { if (!presentValues[i]) throw JsonSyntaxException("Field ${field.name} of ${clazz.qualifiedName} is missing") - if (!param.type.isMarkedNullable) throw JsonSyntaxException("Field ${field.name} of ${clazz.qualifiedName} does not accept nulls") + if (!param.isMarkedNullable) throw JsonSyntaxException("Field ${field.name} of ${clazz.qualifiedName} does not accept nulls") } } @@ -427,7 +441,6 @@ class FactoryAdapter private constructor( stringInterner = stringInterner, aliases = aliases, logMisses = logMisses, - elements = Starbound.ELEMENTS_ADAPTER ) } diff --git a/src/main/kotlin/ru/dbotthepony/kstarbound/network/packets/EntityMessagePacket.kt b/src/main/kotlin/ru/dbotthepony/kstarbound/network/packets/EntityMessagePacket.kt index 5178de79..53ab7778 100644 --- a/src/main/kotlin/ru/dbotthepony/kstarbound/network/packets/EntityMessagePacket.kt +++ b/src/main/kotlin/ru/dbotthepony/kstarbound/network/packets/EntityMessagePacket.kt @@ -45,23 +45,23 @@ class EntityMessagePacket(val entity: Either, val message: String, } private fun handle(connection: Connection, world: World<*, *>) { - if (entity.isLeft) { - val entity = world.entities[entity.left()] - - if (entity == null) { - connection.send(EntityMessageResponsePacket(Either.left("No such entity ${this@EntityMessagePacket.entity}"), id)) - } else { - entity.dispatchMessage(connection.connectionID, message, arguments) - .thenAccept(Consumer { - connection.send(EntityMessageResponsePacket(Either.right(it), id)) - }) - .exceptionally(Function { - connection.send(EntityMessageResponsePacket(Either.left(it.message ?: "Internal server error"), id)) - null - }) - } + val entity = if (entity.isLeft) { + world.entities[entity.left()] } else { - TODO("messages to unique entities") + world.entities.values.firstOrNull { it.uniqueID.get() == entity.right() } + } + + if (entity == null) { + connection.send(EntityMessageResponsePacket(Either.left("No such entity ${this@EntityMessagePacket.entity}"), id)) + } else { + entity.dispatchMessage(connection.connectionID, message, arguments) + .thenAccept(Consumer { + connection.send(EntityMessageResponsePacket(Either.right(it), id)) + }) + .exceptionally(Function { + connection.send(EntityMessageResponsePacket(Either.left(it.message ?: "Internal server error"), id)) + null + }) } } diff --git a/src/main/kotlin/ru/dbotthepony/kstarbound/server/ServerConnection.kt b/src/main/kotlin/ru/dbotthepony/kstarbound/server/ServerConnection.kt index fbdef3bf..508c674e 100644 --- a/src/main/kotlin/ru/dbotthepony/kstarbound/server/ServerConnection.kt +++ b/src/main/kotlin/ru/dbotthepony/kstarbound/server/ServerConnection.kt @@ -5,6 +5,7 @@ import io.netty.channel.ChannelHandlerContext import it.unimi.dsi.fastutil.objects.ObjectOpenHashSet import kotlinx.coroutines.Job import kotlinx.coroutines.channels.Channel +import kotlinx.coroutines.coroutineScope import kotlinx.coroutines.delay import kotlinx.coroutines.future.await import kotlinx.coroutines.launch @@ -38,6 +39,7 @@ import ru.dbotthepony.kstarbound.server.world.WorldStorage import ru.dbotthepony.kstarbound.server.world.LegacyWorldStorage import ru.dbotthepony.kstarbound.server.world.ServerSystemWorld import ru.dbotthepony.kstarbound.server.world.ServerWorld +import ru.dbotthepony.kstarbound.util.ActionPacer import ru.dbotthepony.kstarbound.world.SystemWorldLocation import ru.dbotthepony.kstarbound.world.UniversePos import java.util.UUID @@ -399,6 +401,8 @@ class ServerConnection(val server: StarboundServer, type: ConnectionType) : Conn } } + private val celestialRequestsBudget = ActionPacer(64, 512) + private suspend fun handleCelestialRequests(requests: Collection>) { val responses = ArrayList>() @@ -411,21 +415,26 @@ class ServerConnection(val server: StarboundServer, type: ConnectionType) : Conn val systemParameters = HashMap() val planets = HashMap>() - for (system in systems) { - systemParameters[system.location] = server.universe.parameters(system) ?: continue + coroutineScope { + for (system in systems) { + launch { + celestialRequestsBudget.consume() + systemParameters[system.location] = server.universe.parameters(system) ?: return@launch - val systemPlanets = HashMap() - planets[system.location] = systemPlanets + val systemPlanets = HashMap() + planets[system.location] = systemPlanets - for (planetPos in server.universe.children(system)) { - val parameters = server.universe.parameters(planetPos) ?: continue - val satelliteMap = HashMap() + for (planetPos in server.universe.children(system)) { + val parameters = server.universe.parameters(planetPos) ?: continue + val satelliteMap = HashMap() - for (satellitePos in server.universe.children(planetPos)) { - satelliteMap[satellitePos.satelliteOrbit] = server.universe.parameters(satellitePos) ?: continue + for (satellitePos in server.universe.children(planetPos)) { + satelliteMap[satellitePos.satelliteOrbit] = server.universe.parameters(satellitePos) ?: continue + } + + systemPlanets[planetPos.planetOrbit] = CelestialResponsePacket.PlanetData(parameters, satelliteMap) + } } - - systemPlanets[planetPos.planetOrbit] = CelestialResponsePacket.PlanetData(parameters, satelliteMap) } } @@ -433,6 +442,9 @@ class ServerConnection(val server: StarboundServer, type: ConnectionType) : Conn chunkPos, constellations, systemParameters, planets ))) } else { + // even if malicious actor will do spread-out requests, which will generate new chunk each time + // we still handle requests sequentially + celestialRequestsBudget.consume() val systemPos = UniversePos(request.right()) val map = HashMap() diff --git a/src/main/kotlin/ru/dbotthepony/kstarbound/server/world/ServerUniverse.kt b/src/main/kotlin/ru/dbotthepony/kstarbound/server/world/ServerUniverse.kt index f63d706f..a0422a94 100644 --- a/src/main/kotlin/ru/dbotthepony/kstarbound/server/world/ServerUniverse.kt +++ b/src/main/kotlin/ru/dbotthepony/kstarbound/server/world/ServerUniverse.kt @@ -42,6 +42,7 @@ import ru.dbotthepony.kstarbound.util.random.AbstractPerlinNoise import ru.dbotthepony.kstarbound.util.random.nextRange import ru.dbotthepony.kstarbound.util.random.random import ru.dbotthepony.kstarbound.util.random.staticRandom64 +import ru.dbotthepony.kstarbound.util.supplyAsync import ru.dbotthepony.kstarbound.world.Universe import ru.dbotthepony.kstarbound.world.UniversePos import java.io.Closeable @@ -181,15 +182,6 @@ class ServerUniverse(folder: File? = null) : Universe(), Closeable { } private data class System(val x: Int, val y: Int, val z: Int, val parameters: CelestialParameters, val planets: Map, CelestialParameters>) { - constructor(x: Int, y: Int, z: Int, data: ResultSet) : this( - x, y, z, - Starbound.gson.fromJson(data.getBytes(1).readJsonElementInflated())!!, - data.getBytes(2).readJsonArrayInflated().associate { - it as JsonArray - (it[0].asInt to it[1].asInt) to Starbound.gson.fromJson(it[2])!! - } - ) - fun parameters(pos: UniversePos): CelestialParameters? { if (pos.isSystem) { return parameters @@ -266,14 +258,27 @@ class ServerUniverse(folder: File? = null) : Universe(), Closeable { .executor(Starbound.EXECUTOR) .build>() - private fun loadSystem(pos: Vector3i): System? { + private fun loadSystem(pos: Vector3i): CompletableFuture? { selectSystem.setInt(1, pos.x) selectSystem.setInt(2, pos.y) selectSystem.setInt(3, pos.z) return selectSystem.executeQuery().use { if (it.next()) { - System(pos.x, pos.y, pos.z, it) + val parametersBytes = it.getBytes(1) + val planetsBytes = it.getBytes(2) + + // deserialize in off-thread since it involves big json structures + Starbound.EXECUTOR.supplyAsync { + val parameters: CelestialParameters = Starbound.gson.fromJson(parametersBytes.readJsonElementInflated())!! + + val planets: Map, CelestialParameters> = planetsBytes.readJsonArrayInflated().associate { + it as JsonArray + (it[0].asInt to it[1].asInt) to Starbound.gson.fromJson(it[2])!! + } + + System(pos.x, pos.y, pos.z, parameters, planets) + } } else { null } @@ -285,8 +290,9 @@ class ServerUniverse(folder: File? = null) : Universe(), Closeable { if (existing != null) { // hit, system already exists + val wait = existing.await() systemFutures.remove(pos) - return existing + return wait } // lets try to get chunk this system is in @@ -298,7 +304,7 @@ class ServerUniverse(folder: File? = null) : Universe(), Closeable { if (pos !in chunk.systems) return null - return loadSystem(pos) + return loadSystem(pos)!!.await() } private fun getSystem(pos: Vector3i): CompletableFuture {