From 8a4a84c05bc8db8de8279740e3d4b99a75924a29 Mon Sep 17 00:00:00 2001 From: DBotThePony Date: Mon, 6 Feb 2023 20:29:18 +0700 Subject: [PATCH] BinaryJsonReader --- .../dbotthepony/kstarbound/io/StarboundPak.kt | 4 +- .../kstarbound/io/json/BinaryJson.kt | 193 ------- .../kstarbound/io/json/BinaryJsonReader.kt | 486 ++++++++++++++++++ 3 files changed, 488 insertions(+), 195 deletions(-) delete mode 100644 src/main/kotlin/ru/dbotthepony/kstarbound/io/json/BinaryJson.kt create mode 100644 src/main/kotlin/ru/dbotthepony/kstarbound/io/json/BinaryJsonReader.kt diff --git a/src/main/kotlin/ru/dbotthepony/kstarbound/io/StarboundPak.kt b/src/main/kotlin/ru/dbotthepony/kstarbound/io/StarboundPak.kt index 2a90f9fa..7c8541ff 100644 --- a/src/main/kotlin/ru/dbotthepony/kstarbound/io/StarboundPak.kt +++ b/src/main/kotlin/ru/dbotthepony/kstarbound/io/StarboundPak.kt @@ -3,7 +3,7 @@ package ru.dbotthepony.kstarbound.io import it.unimi.dsi.fastutil.objects.Object2ObjectFunction import it.unimi.dsi.fastutil.objects.Object2ObjectOpenHashMap import ru.dbotthepony.kstarbound.api.IStarboundFile -import ru.dbotthepony.kstarbound.io.json.BinaryJson +import ru.dbotthepony.kstarbound.io.json.BinaryJsonReader import java.io.BufferedInputStream import java.io.Closeable import java.io.DataInputStream @@ -180,7 +180,7 @@ class StarboundPak(val path: File, callback: (finished: Boolean, status: String) } // сразу за INDEX идут метаданные в формате Binary Json - val metadata = BinaryJson.readObject(reader) + val metadata = BinaryJsonReader.readObject(reader) // сразу за метаданными идёт количество файлов внутри данного pak в формате Big Endian variable int val indexNodeCount = reader.readVarLong() diff --git a/src/main/kotlin/ru/dbotthepony/kstarbound/io/json/BinaryJson.kt b/src/main/kotlin/ru/dbotthepony/kstarbound/io/json/BinaryJson.kt deleted file mode 100644 index e719ffe0..00000000 --- a/src/main/kotlin/ru/dbotthepony/kstarbound/io/json/BinaryJson.kt +++ /dev/null @@ -1,193 +0,0 @@ -package ru.dbotthepony.kstarbound.io.json - -import com.google.gson.* -import ru.dbotthepony.kstarbound.io.readASCIIString -import ru.dbotthepony.kstarbound.io.readVarInt -import ru.dbotthepony.kstarbound.io.readVarLong -import java.io.DataInputStream -import java.io.RandomAccessFile - -/** - * Represents interface to read binary json format by Chucklefish - */ -object BinaryJson { - const val TYPE_NULL = 0x01 - const val TYPE_FLOAT = 0x02 - const val TYPE_DOUBLE = TYPE_FLOAT - const val TYPE_BOOLEAN = 0x03 - - /** - * На самом деле, variable int - */ - const val TYPE_INT = 0x04 - const val TYPE_STRING = 0x05 - const val TYPE_ARRAY = 0x06 - const val TYPE_OBJECT = 0x07 - - fun readElement(reader: RandomAccessFile): JsonElement { - return when (val id = reader.read()) { - TYPE_NULL -> JsonNull.INSTANCE - TYPE_DOUBLE -> JsonPrimitive(reader.readDouble()) - TYPE_BOOLEAN -> JsonPrimitive(reader.readBoolean()) - TYPE_INT -> { - var read = reader.readVarLong() - val sign = read and 0x1L - read = read ushr 1 - - if (sign == 1L) { - JsonPrimitive(read - 1L) - } else { - JsonPrimitive(read) - } - } - TYPE_STRING -> JsonPrimitive(reader.readASCIIString(reader.readVarInt())) - TYPE_ARRAY -> readArray(reader) - TYPE_OBJECT -> readObject(reader) - else -> throw JsonParseException("Unknown element type $id") - } - } - - fun readElement(reader: DataInputStream): JsonElement { - return when (val id = reader.read()) { - TYPE_NULL -> JsonNull.INSTANCE - TYPE_DOUBLE -> JsonPrimitive(reader.readDouble()) - TYPE_BOOLEAN -> JsonPrimitive(reader.readBoolean()) - TYPE_INT -> { - var read = reader.readVarLong() - val sign = read and 0x1L - read = read ushr 1 - - if (sign == 1L) { - JsonPrimitive(read - 1L) - } else { - JsonPrimitive(read) - } - } - TYPE_STRING -> JsonPrimitive(reader.readASCIIString(reader.readVarInt())) - TYPE_ARRAY -> readArray(reader) - TYPE_OBJECT -> readObject(reader) - else -> throw JsonParseException("Unknown element type $id") - } - } - - fun readObject(reader: RandomAccessFile): JsonObject { - val values = reader.readVarInt() - 1 - - if (values == -1) { - return JsonObject() - } - - if (values < -1) { - throw JsonParseException("Tried to read json object with $values elements in it") - } - - val build = JsonObject() - - for (i in 0 .. values) { - val key: String - - try { - key = reader.readASCIIString(reader.readVarInt()) - } catch(err: Throwable) { - throw JsonParseException("Reading json object at $i", err) - } - - try { - build.add(key, readElement(reader)) - } catch(err: Throwable) { - throw JsonParseException("Reading json object at $i with name $key", err) - } - } - - return build - } - - fun readObject(reader: DataInputStream): JsonObject { - val values = reader.readVarInt() - 1 - - if (values == -1) { - return JsonObject() - } - - if (values < -1) { - throw JsonParseException("Tried to read json object with $values elements in it") - } - - val build = JsonObject() - - for (i in 0 .. values) { - val key: String - - try { - key = reader.readASCIIString(reader.readVarInt()) - } catch(err: Throwable) { - throw JsonParseException("Reading json object at $i", err) - } - - try { - build.add(key, readElement(reader)) - } catch(err: Throwable) { - throw JsonParseException("Reading json object at $i with name $key", err) - } - } - - return build - } - - fun readArray(reader: RandomAccessFile): JsonArray { - val values = reader.readVarInt() - 1 - - if (values == -1) { - return JsonArray() - } - - if (values < -1) { - throw JsonParseException("Tried to read json array with $values elements in it") - } - - val build = JsonArray() - - for (i in 0 .. values) { - build.add(readElement(reader)) - } - - return build - } - - fun readArray(reader: DataInputStream): JsonArray { - val values = reader.readVarInt() - 1 - - if (values == -1) { - return JsonArray() - } - - if (values < -1) { - throw JsonParseException("Tried to read json array with $values elements in it") - } - - val build = JsonArray() - - for (i in 0 .. values) { - build.add(readElement(reader)) - } - - return build - } -} - -class VersionedJSON(var name: String = "Versioned JSON") { - var isVersioned = false - var version = 0 - var data: JsonElement? = null - - constructor(stream: DataInputStream) : this() { - name = stream.readASCIIString(stream.readVarInt()) - isVersioned = stream.readBoolean() - - if (isVersioned) { - version = stream.readInt() - } - - data = BinaryJson.readElement(stream) - } -} diff --git a/src/main/kotlin/ru/dbotthepony/kstarbound/io/json/BinaryJsonReader.kt b/src/main/kotlin/ru/dbotthepony/kstarbound/io/json/BinaryJsonReader.kt new file mode 100644 index 00000000..4ac8ebca --- /dev/null +++ b/src/main/kotlin/ru/dbotthepony/kstarbound/io/json/BinaryJsonReader.kt @@ -0,0 +1,486 @@ +package ru.dbotthepony.kstarbound.io.json + +import com.google.gson.JsonArray +import com.google.gson.JsonElement +import com.google.gson.JsonNull +import com.google.gson.JsonObject +import com.google.gson.JsonParseException +import com.google.gson.JsonPrimitive +import com.google.gson.JsonSyntaxException +import com.google.gson.stream.JsonReader +import com.google.gson.stream.JsonToken +import ru.dbotthepony.kstarbound.io.readASCIIString +import ru.dbotthepony.kstarbound.io.readVarInt +import ru.dbotthepony.kstarbound.io.readVarLong +import java.io.DataInputStream +import java.io.EOFException +import java.io.InputStream +import java.io.RandomAccessFile +import java.io.Reader +import java.util.LinkedList + +class BinaryJsonReader(private val stream: DataInputStream) : JsonReader(unreadable) { + constructor(stream: InputStream) : this(DataInputStream(stream)) + + private var needToReadNext = true + + private fun popstack(check: NothingReader) { + val popped = stack.removeLast() + check(popped == check) { "$popped != $check" } + needToReadNext = true + } + + private fun actuallyReadNext(expectingValidValue: Boolean) { + when (val id = stream.read()) { + TYPE_NULL -> stack.addLast(nullReader) + TYPE_DOUBLE -> stack.addLast(DoubleReader(stream.readDouble())) + TYPE_BOOLEAN -> stack.addLast(if (stream.readBoolean()) trueReader else falseReader) + TYPE_INT -> stack.addLast(LongReader(fixSignedInt(stream.readVarLong()))) + TYPE_STRING -> stack.addLast(StringReader(stream.readVarInt())) + TYPE_OBJECT -> stack.addLast(ObjectReader(stream.readVarInt())) + TYPE_ARRAY -> stack.addLast(ArrayReader(stream.readVarInt())) + -1 -> { if (expectingValidValue) throw EOFException("Unexpected end of stream while expecting binary json element") } + else -> throw IllegalArgumentException("Invalid type ID: $id") + } + } + + private fun readNext() { + if (!needToReadNext) + return + + needToReadNext = false + val last = stack.lastOrNull() + + if (last is ObjectReader) { + if (last.readPairs == last.pairs) { + stack.removeLast() + stack.addLast(endObject) + return + } + + if (last.readingName) { + last.readingName = false + stack.addLast(NameReader(stream.readASCIIString(stream.readVarInt()))) + } else { + last.readingName = true + last.readPairs++ + actuallyReadNext(true) + } + } else if (last is ArrayReader) { + if (last.readValues == last.size) { + stack.removeLast() + stack.addLast(endArray) + return + } + + last.readValues++ + actuallyReadNext(true) + } else { + actuallyReadNext(false) + } + } + + override fun peek(): JsonToken { + readNext() + return (stack.lastOrNull() ?: endReader).peek() + } + + override fun hasNext(): Boolean { + return peek() != JsonToken.END_OBJECT && peek() != JsonToken.END_ARRAY + } + + private abstract inner class NothingReader(val name: String, val token: JsonToken) { + open fun beginArray(): Unit = throw JsonSyntaxException("Expected BEGIN_ARRAY, got $name") + open fun endArray(): Unit = throw JsonSyntaxException("Expected END_ARRAY, got $name") + open fun beginObject(): Unit = throw JsonSyntaxException("Expected BEGIN_OBJECT, got $name") + open fun endObject(): Unit = throw JsonSyntaxException("Expected END_OBJECT, got $name") + open fun nextName(): String = throw JsonSyntaxException("Expected a name, got $name") + open fun nextString(): String = throw JsonSyntaxException("Expected a string, got $name") + open fun nextBoolean(): Boolean = throw JsonSyntaxException("Expected a boolean, got $name") + open fun nextNull(): Unit = throw JsonSyntaxException("Expected a null, got $name") + open fun nextDouble(): Double = throw JsonSyntaxException("Expected a double, got $name") + open fun nextLong(): Long = throw JsonSyntaxException("Expected a long, got $name") + open fun nextInt(): Int = throw JsonSyntaxException("Expected an int, got $name") + + open fun skipValue() { + popstack(this) + } + + open fun peek(): JsonToken = token + + override fun toString(): String { + return "ReaderShard[$name, $token]" + } + } + + private inner class NullReader : NothingReader("null", JsonToken.NULL) { + override fun nextNull() { popstack(this) } + } + + private inner class TrueReader : NothingReader("boolean", JsonToken.BOOLEAN) { + override fun nextBoolean(): Boolean { popstack(this); return true } + } + + private inner class FalseReader : NothingReader("boolean", JsonToken.BOOLEAN) { + override fun nextBoolean(): Boolean { popstack(this); return false } + } + + private inner class EndReader : NothingReader("end_document", JsonToken.END_DOCUMENT) { + override fun skipValue() { + check(stack.size == 0) + } + } + + private val nullReader = NullReader() + private val trueReader = TrueReader() + private val falseReader = FalseReader() + private val endReader = EndReader() + + private inner class LongReader(val value: Long) : NothingReader("long", JsonToken.NUMBER) { + override fun nextLong(): Long { + popstack(this) + return value + } + + override fun nextDouble() = nextLong().toDouble() + override fun nextInt() = nextLong().toInt() + override fun nextString() = nextLong().toString() + } + + private inner class DoubleReader(val value: Double) : NothingReader("double", JsonToken.NUMBER) { + override fun nextDouble(): Double { + popstack(this) + return value + } + + override fun nextInt() = nextDouble().toInt() + override fun nextLong() = nextDouble().toLong() + override fun nextString() = nextDouble().toString() + } + + private inner class StringReader(val length: Int) : NothingReader("string", JsonToken.STRING) { + private val value by lazy(LazyThreadSafetyMode.NONE) { stream.readASCIIString(length) } + + override fun skipValue() { + popstack(this) + stream.skipNBytes(length.toLong()) + } + + override fun nextString(): String { + popstack(this) + return value + } + + override fun nextInt(): Int { + try { + return value.toInt().also { popstack(this) } + } catch(_: NumberFormatException) { + try { + return value.toDouble().toInt().also { popstack(this) } + } catch (_: NumberFormatException) { + + } + } + + throw JsonSyntaxException("Expected an int, got string near ${this@BinaryJsonReader.path}") + } + + override fun nextLong(): Long { + try { + return value.toLong().also { popstack(this) } + } catch(_: NumberFormatException) { + try { + return value.toDouble().toLong().also { popstack(this) } + } catch (_: NumberFormatException) { + + } + } + + throw JsonSyntaxException("Expected an long, got string near ${this@BinaryJsonReader.path}") + } + + override fun nextDouble(): Double { + try { + return value.toDouble().also { popstack(this) } + } catch (_: NumberFormatException) { + + } + + throw JsonSyntaxException("Expected an long, got string near ${this@BinaryJsonReader.path}") + } + } + + private inner class NameReader(val value: String) : NothingReader("name", JsonToken.NAME) { + override fun nextName(): String { + popstack(this) + return value + } + } + + private inner class ObjectReader(val pairs: Int) : NothingReader("object", JsonToken.BEGIN_OBJECT) { + var readingName = true + var readPairs = 0 + + var consumed = false + + override fun beginObject() { + check(!consumed) { "Already called beginObject on this object" } + check(stack.last() === this) + consumed = true + needToReadNext = true + } + } + + private inner class EndObject : NothingReader("end object", JsonToken.END_OBJECT) { + override fun endObject() { + popstack(this) + } + } + + private val endObject = EndObject() + + private inner class ArrayReader(val size: Int) : NothingReader("array", JsonToken.BEGIN_ARRAY) { + var readValues = 0 + var consumed = false + + override fun beginArray() { + check(!consumed) { "Already called beginArray on this array" } + check(stack.last() === this) + consumed = true + needToReadNext = true + } + } + + private inner class EndArray : NothingReader("end array", JsonToken.END_ARRAY) { + override fun endArray() { + popstack(this) + } + } + + override fun close() { + stack.clear() + stream.close() + } + + override fun getPath(): String { + return "" + } + + private val endArray = EndArray() + + private val stack = LinkedList() + + override fun nextDouble(): Double { + readNext() + return (stack.lastOrNull() ?: endReader).nextDouble() + } + + override fun nextLong(): Long { + readNext() + return (stack.lastOrNull() ?: endReader).nextLong() + } + + override fun nextInt(): Int { + readNext() + return (stack.lastOrNull() ?: endReader).nextInt() + } + + override fun nextNull() { + readNext() + return (stack.lastOrNull() ?: endReader).nextNull() + } + + override fun beginObject() { + readNext() + return (stack.lastOrNull() ?: endReader).beginObject() + } + + override fun endObject() { + readNext() + return (stack.lastOrNull() ?: endReader).endObject() + } + + override fun beginArray() { + readNext() + return (stack.lastOrNull() ?: endReader).beginArray() + } + + override fun endArray() { + readNext() + return (stack.lastOrNull() ?: endReader).endArray() + } + + override fun nextName(): String { + readNext() + return (stack.lastOrNull() ?: endReader).nextName() + } + + override fun nextString(): String { + readNext() + return (stack.lastOrNull() ?: endReader).nextString() + } + + override fun nextBoolean(): Boolean { + readNext() + return (stack.lastOrNull() ?: endReader).nextBoolean() + } + + override fun skipValue() { + readNext() + return (stack.lastOrNull() ?: endReader).skipValue() + } + + companion object { + const val TYPE_NULL = 0x01 + const val TYPE_DOUBLE = 0x02 + const val TYPE_BOOLEAN = 0x03 + + /** + * На самом деле, variable int + */ + const val TYPE_INT = 0x04 + const val TYPE_STRING = 0x05 + const val TYPE_ARRAY = 0x06 + const val TYPE_OBJECT = 0x07 + + private fun fixSignedInt(read: Long): Long { + val sign = read and 0x1L + @Suppress("name_shadowing") + val read = read ushr 1 + + if (sign == 1L) { + return -read - 1L + } else { + return read + } + } + + /** + * Позволяет читать двоичный JSON прямиком в [JsonElement] + */ + fun readElement(reader: RandomAccessFile): JsonElement { + return when (val id = reader.read()) { + TYPE_NULL -> JsonNull.INSTANCE + TYPE_DOUBLE -> JsonPrimitive(reader.readDouble()) + TYPE_BOOLEAN -> JsonPrimitive(reader.readBoolean()) + TYPE_INT -> JsonPrimitive(fixSignedInt(reader.readVarLong())) + TYPE_STRING -> JsonPrimitive(reader.readASCIIString(reader.readVarInt())) + TYPE_ARRAY -> readArray(reader) + TYPE_OBJECT -> readObject(reader) + else -> throw JsonParseException("Unknown element type $id") + } + } + + /** + * Позволяет читать двоичный JSON прямиком в [JsonElement] + */ + fun readElement(reader: DataInputStream): JsonElement { + return when (val id = reader.read()) { + TYPE_NULL -> JsonNull.INSTANCE + TYPE_DOUBLE -> JsonPrimitive(reader.readDouble()) + TYPE_BOOLEAN -> JsonPrimitive(reader.readBoolean()) + TYPE_INT -> JsonPrimitive(fixSignedInt(reader.readVarLong())) + TYPE_STRING -> JsonPrimitive(reader.readASCIIString(reader.readVarInt())) + TYPE_ARRAY -> readArray(reader) + TYPE_OBJECT -> readObject(reader) + else -> throw JsonParseException("Unknown element type $id") + } + } + + /** + * Позволяет читать двоичный JSON объект прямиком в [JsonObject] + */ + fun readObject(reader: RandomAccessFile): JsonObject { + val values = reader.readVarInt() - 1 + if (values == -1) return JsonObject() + if (values < -1) throw JsonSyntaxException("Tried to read json object with $values elements in it") + + val build = JsonObject() + + for (i in 0 .. values) { + val key: String + + try { + key = reader.readASCIIString(reader.readVarInt()) + } catch(err: Throwable) { + throw JsonSyntaxException("Reading json object at $i", err) + } + + try { + build.add(key, readElement(reader)) + } catch(err: Throwable) { + throw JsonSyntaxException("Reading json object at $i with name $key", err) + } + } + + return build + } + + /** + * Позволяет читать двоичный JSON объект прямиком в [JsonObject] + */ + fun readObject(reader: DataInputStream): JsonObject { + val values = reader.readVarInt() - 1 + if (values == -1) return JsonObject() + if (values < -1) throw JsonSyntaxException("Tried to read json object with $values elements in it") + + val build = JsonObject() + + for (i in 0 .. values) { + val key: String + + try { + key = reader.readASCIIString(reader.readVarInt()) + } catch(err: Throwable) { + throw JsonSyntaxException("Reading json object at $i", err) + } + + try { + build.add(key, readElement(reader)) + } catch(err: Throwable) { + throw JsonSyntaxException("Reading json object at $i with name $key", err) + } + } + + return build + } + + /** + * Позволяет читать двоичный JSON массив прямиком в [JsonArray] + */ + fun readArray(reader: RandomAccessFile): JsonArray { + val values = reader.readVarInt() - 1 + + if (values == -1) return JsonArray() + if (values < -1) throw JsonSyntaxException("Tried to read json array with $values elements in it") + + val build = JsonArray(values) + for (i in 0 .. values) build.add(readElement(reader)) + return build + } + + /** + * Позволяет читать двоичный JSON массив прямиком в [JsonArray] + */ + fun readArray(reader: DataInputStream): JsonArray { + val values = reader.readVarInt() - 1 + + if (values == -1) return JsonArray() + if (values < -1) throw JsonSyntaxException("Tried to read json array with $values elements in it") + + val build = JsonArray(values) + for (i in 0 .. values) build.add(readElement(reader)) + return build + } + + private val unreadable = object : Reader() { + override fun read(cbuf: CharArray, off: Int, len: Int): Int { + throw AssertionError() + } + + override fun close() { + throw AssertionError() + } + } + } +}