BinaryJsonReader

This commit is contained in:
DBotThePony 2023-02-06 20:29:18 +07:00
parent 6aca9d40f1
commit 8a4a84c05b
Signed by: DBot
GPG Key ID: DCC23B5715498507
3 changed files with 488 additions and 195 deletions

View File

@ -3,7 +3,7 @@ package ru.dbotthepony.kstarbound.io
import it.unimi.dsi.fastutil.objects.Object2ObjectFunction
import it.unimi.dsi.fastutil.objects.Object2ObjectOpenHashMap
import ru.dbotthepony.kstarbound.api.IStarboundFile
import ru.dbotthepony.kstarbound.io.json.BinaryJson
import ru.dbotthepony.kstarbound.io.json.BinaryJsonReader
import java.io.BufferedInputStream
import java.io.Closeable
import java.io.DataInputStream
@ -180,7 +180,7 @@ class StarboundPak(val path: File, callback: (finished: Boolean, status: String)
}
// сразу за INDEX идут метаданные в формате Binary Json
val metadata = BinaryJson.readObject(reader)
val metadata = BinaryJsonReader.readObject(reader)
// сразу за метаданными идёт количество файлов внутри данного pak в формате Big Endian variable int
val indexNodeCount = reader.readVarLong()

View File

@ -1,193 +0,0 @@
package ru.dbotthepony.kstarbound.io.json
import com.google.gson.*
import ru.dbotthepony.kstarbound.io.readASCIIString
import ru.dbotthepony.kstarbound.io.readVarInt
import ru.dbotthepony.kstarbound.io.readVarLong
import java.io.DataInputStream
import java.io.RandomAccessFile
/**
* Represents interface to read binary json format by Chucklefish
*/
object BinaryJson {
const val TYPE_NULL = 0x01
const val TYPE_FLOAT = 0x02
const val TYPE_DOUBLE = TYPE_FLOAT
const val TYPE_BOOLEAN = 0x03
/**
* На самом деле, variable int
*/
const val TYPE_INT = 0x04
const val TYPE_STRING = 0x05
const val TYPE_ARRAY = 0x06
const val TYPE_OBJECT = 0x07
fun readElement(reader: RandomAccessFile): JsonElement {
return when (val id = reader.read()) {
TYPE_NULL -> JsonNull.INSTANCE
TYPE_DOUBLE -> JsonPrimitive(reader.readDouble())
TYPE_BOOLEAN -> JsonPrimitive(reader.readBoolean())
TYPE_INT -> {
var read = reader.readVarLong()
val sign = read and 0x1L
read = read ushr 1
if (sign == 1L) {
JsonPrimitive(read - 1L)
} else {
JsonPrimitive(read)
}
}
TYPE_STRING -> JsonPrimitive(reader.readASCIIString(reader.readVarInt()))
TYPE_ARRAY -> readArray(reader)
TYPE_OBJECT -> readObject(reader)
else -> throw JsonParseException("Unknown element type $id")
}
}
fun readElement(reader: DataInputStream): JsonElement {
return when (val id = reader.read()) {
TYPE_NULL -> JsonNull.INSTANCE
TYPE_DOUBLE -> JsonPrimitive(reader.readDouble())
TYPE_BOOLEAN -> JsonPrimitive(reader.readBoolean())
TYPE_INT -> {
var read = reader.readVarLong()
val sign = read and 0x1L
read = read ushr 1
if (sign == 1L) {
JsonPrimitive(read - 1L)
} else {
JsonPrimitive(read)
}
}
TYPE_STRING -> JsonPrimitive(reader.readASCIIString(reader.readVarInt()))
TYPE_ARRAY -> readArray(reader)
TYPE_OBJECT -> readObject(reader)
else -> throw JsonParseException("Unknown element type $id")
}
}
fun readObject(reader: RandomAccessFile): JsonObject {
val values = reader.readVarInt() - 1
if (values == -1) {
return JsonObject()
}
if (values < -1) {
throw JsonParseException("Tried to read json object with $values elements in it")
}
val build = JsonObject()
for (i in 0 .. values) {
val key: String
try {
key = reader.readASCIIString(reader.readVarInt())
} catch(err: Throwable) {
throw JsonParseException("Reading json object at $i", err)
}
try {
build.add(key, readElement(reader))
} catch(err: Throwable) {
throw JsonParseException("Reading json object at $i with name $key", err)
}
}
return build
}
fun readObject(reader: DataInputStream): JsonObject {
val values = reader.readVarInt() - 1
if (values == -1) {
return JsonObject()
}
if (values < -1) {
throw JsonParseException("Tried to read json object with $values elements in it")
}
val build = JsonObject()
for (i in 0 .. values) {
val key: String
try {
key = reader.readASCIIString(reader.readVarInt())
} catch(err: Throwable) {
throw JsonParseException("Reading json object at $i", err)
}
try {
build.add(key, readElement(reader))
} catch(err: Throwable) {
throw JsonParseException("Reading json object at $i with name $key", err)
}
}
return build
}
fun readArray(reader: RandomAccessFile): JsonArray {
val values = reader.readVarInt() - 1
if (values == -1) {
return JsonArray()
}
if (values < -1) {
throw JsonParseException("Tried to read json array with $values elements in it")
}
val build = JsonArray()
for (i in 0 .. values) {
build.add(readElement(reader))
}
return build
}
fun readArray(reader: DataInputStream): JsonArray {
val values = reader.readVarInt() - 1
if (values == -1) {
return JsonArray()
}
if (values < -1) {
throw JsonParseException("Tried to read json array with $values elements in it")
}
val build = JsonArray()
for (i in 0 .. values) {
build.add(readElement(reader))
}
return build
}
}
class VersionedJSON(var name: String = "Versioned JSON") {
var isVersioned = false
var version = 0
var data: JsonElement? = null
constructor(stream: DataInputStream) : this() {
name = stream.readASCIIString(stream.readVarInt())
isVersioned = stream.readBoolean()
if (isVersioned) {
version = stream.readInt()
}
data = BinaryJson.readElement(stream)
}
}

View File

@ -0,0 +1,486 @@
package ru.dbotthepony.kstarbound.io.json
import com.google.gson.JsonArray
import com.google.gson.JsonElement
import com.google.gson.JsonNull
import com.google.gson.JsonObject
import com.google.gson.JsonParseException
import com.google.gson.JsonPrimitive
import com.google.gson.JsonSyntaxException
import com.google.gson.stream.JsonReader
import com.google.gson.stream.JsonToken
import ru.dbotthepony.kstarbound.io.readASCIIString
import ru.dbotthepony.kstarbound.io.readVarInt
import ru.dbotthepony.kstarbound.io.readVarLong
import java.io.DataInputStream
import java.io.EOFException
import java.io.InputStream
import java.io.RandomAccessFile
import java.io.Reader
import java.util.LinkedList
class BinaryJsonReader(private val stream: DataInputStream) : JsonReader(unreadable) {
constructor(stream: InputStream) : this(DataInputStream(stream))
private var needToReadNext = true
private fun popstack(check: NothingReader) {
val popped = stack.removeLast()
check(popped == check) { "$popped != $check" }
needToReadNext = true
}
private fun actuallyReadNext(expectingValidValue: Boolean) {
when (val id = stream.read()) {
TYPE_NULL -> stack.addLast(nullReader)
TYPE_DOUBLE -> stack.addLast(DoubleReader(stream.readDouble()))
TYPE_BOOLEAN -> stack.addLast(if (stream.readBoolean()) trueReader else falseReader)
TYPE_INT -> stack.addLast(LongReader(fixSignedInt(stream.readVarLong())))
TYPE_STRING -> stack.addLast(StringReader(stream.readVarInt()))
TYPE_OBJECT -> stack.addLast(ObjectReader(stream.readVarInt()))
TYPE_ARRAY -> stack.addLast(ArrayReader(stream.readVarInt()))
-1 -> { if (expectingValidValue) throw EOFException("Unexpected end of stream while expecting binary json element") }
else -> throw IllegalArgumentException("Invalid type ID: $id")
}
}
private fun readNext() {
if (!needToReadNext)
return
needToReadNext = false
val last = stack.lastOrNull()
if (last is ObjectReader) {
if (last.readPairs == last.pairs) {
stack.removeLast()
stack.addLast(endObject)
return
}
if (last.readingName) {
last.readingName = false
stack.addLast(NameReader(stream.readASCIIString(stream.readVarInt())))
} else {
last.readingName = true
last.readPairs++
actuallyReadNext(true)
}
} else if (last is ArrayReader) {
if (last.readValues == last.size) {
stack.removeLast()
stack.addLast(endArray)
return
}
last.readValues++
actuallyReadNext(true)
} else {
actuallyReadNext(false)
}
}
override fun peek(): JsonToken {
readNext()
return (stack.lastOrNull() ?: endReader).peek()
}
override fun hasNext(): Boolean {
return peek() != JsonToken.END_OBJECT && peek() != JsonToken.END_ARRAY
}
private abstract inner class NothingReader(val name: String, val token: JsonToken) {
open fun beginArray(): Unit = throw JsonSyntaxException("Expected BEGIN_ARRAY, got $name")
open fun endArray(): Unit = throw JsonSyntaxException("Expected END_ARRAY, got $name")
open fun beginObject(): Unit = throw JsonSyntaxException("Expected BEGIN_OBJECT, got $name")
open fun endObject(): Unit = throw JsonSyntaxException("Expected END_OBJECT, got $name")
open fun nextName(): String = throw JsonSyntaxException("Expected a name, got $name")
open fun nextString(): String = throw JsonSyntaxException("Expected a string, got $name")
open fun nextBoolean(): Boolean = throw JsonSyntaxException("Expected a boolean, got $name")
open fun nextNull(): Unit = throw JsonSyntaxException("Expected a null, got $name")
open fun nextDouble(): Double = throw JsonSyntaxException("Expected a double, got $name")
open fun nextLong(): Long = throw JsonSyntaxException("Expected a long, got $name")
open fun nextInt(): Int = throw JsonSyntaxException("Expected an int, got $name")
open fun skipValue() {
popstack(this)
}
open fun peek(): JsonToken = token
override fun toString(): String {
return "ReaderShard[$name, $token]"
}
}
private inner class NullReader : NothingReader("null", JsonToken.NULL) {
override fun nextNull() { popstack(this) }
}
private inner class TrueReader : NothingReader("boolean", JsonToken.BOOLEAN) {
override fun nextBoolean(): Boolean { popstack(this); return true }
}
private inner class FalseReader : NothingReader("boolean", JsonToken.BOOLEAN) {
override fun nextBoolean(): Boolean { popstack(this); return false }
}
private inner class EndReader : NothingReader("end_document", JsonToken.END_DOCUMENT) {
override fun skipValue() {
check(stack.size == 0)
}
}
private val nullReader = NullReader()
private val trueReader = TrueReader()
private val falseReader = FalseReader()
private val endReader = EndReader()
private inner class LongReader(val value: Long) : NothingReader("long", JsonToken.NUMBER) {
override fun nextLong(): Long {
popstack(this)
return value
}
override fun nextDouble() = nextLong().toDouble()
override fun nextInt() = nextLong().toInt()
override fun nextString() = nextLong().toString()
}
private inner class DoubleReader(val value: Double) : NothingReader("double", JsonToken.NUMBER) {
override fun nextDouble(): Double {
popstack(this)
return value
}
override fun nextInt() = nextDouble().toInt()
override fun nextLong() = nextDouble().toLong()
override fun nextString() = nextDouble().toString()
}
private inner class StringReader(val length: Int) : NothingReader("string", JsonToken.STRING) {
private val value by lazy(LazyThreadSafetyMode.NONE) { stream.readASCIIString(length) }
override fun skipValue() {
popstack(this)
stream.skipNBytes(length.toLong())
}
override fun nextString(): String {
popstack(this)
return value
}
override fun nextInt(): Int {
try {
return value.toInt().also { popstack(this) }
} catch(_: NumberFormatException) {
try {
return value.toDouble().toInt().also { popstack(this) }
} catch (_: NumberFormatException) {
}
}
throw JsonSyntaxException("Expected an int, got string near ${this@BinaryJsonReader.path}")
}
override fun nextLong(): Long {
try {
return value.toLong().also { popstack(this) }
} catch(_: NumberFormatException) {
try {
return value.toDouble().toLong().also { popstack(this) }
} catch (_: NumberFormatException) {
}
}
throw JsonSyntaxException("Expected an long, got string near ${this@BinaryJsonReader.path}")
}
override fun nextDouble(): Double {
try {
return value.toDouble().also { popstack(this) }
} catch (_: NumberFormatException) {
}
throw JsonSyntaxException("Expected an long, got string near ${this@BinaryJsonReader.path}")
}
}
private inner class NameReader(val value: String) : NothingReader("name", JsonToken.NAME) {
override fun nextName(): String {
popstack(this)
return value
}
}
private inner class ObjectReader(val pairs: Int) : NothingReader("object", JsonToken.BEGIN_OBJECT) {
var readingName = true
var readPairs = 0
var consumed = false
override fun beginObject() {
check(!consumed) { "Already called beginObject on this object" }
check(stack.last() === this)
consumed = true
needToReadNext = true
}
}
private inner class EndObject : NothingReader("end object", JsonToken.END_OBJECT) {
override fun endObject() {
popstack(this)
}
}
private val endObject = EndObject()
private inner class ArrayReader(val size: Int) : NothingReader("array", JsonToken.BEGIN_ARRAY) {
var readValues = 0
var consumed = false
override fun beginArray() {
check(!consumed) { "Already called beginArray on this array" }
check(stack.last() === this)
consumed = true
needToReadNext = true
}
}
private inner class EndArray : NothingReader("end array", JsonToken.END_ARRAY) {
override fun endArray() {
popstack(this)
}
}
override fun close() {
stack.clear()
stream.close()
}
override fun getPath(): String {
return "<binary json>"
}
private val endArray = EndArray()
private val stack = LinkedList<NothingReader>()
override fun nextDouble(): Double {
readNext()
return (stack.lastOrNull() ?: endReader).nextDouble()
}
override fun nextLong(): Long {
readNext()
return (stack.lastOrNull() ?: endReader).nextLong()
}
override fun nextInt(): Int {
readNext()
return (stack.lastOrNull() ?: endReader).nextInt()
}
override fun nextNull() {
readNext()
return (stack.lastOrNull() ?: endReader).nextNull()
}
override fun beginObject() {
readNext()
return (stack.lastOrNull() ?: endReader).beginObject()
}
override fun endObject() {
readNext()
return (stack.lastOrNull() ?: endReader).endObject()
}
override fun beginArray() {
readNext()
return (stack.lastOrNull() ?: endReader).beginArray()
}
override fun endArray() {
readNext()
return (stack.lastOrNull() ?: endReader).endArray()
}
override fun nextName(): String {
readNext()
return (stack.lastOrNull() ?: endReader).nextName()
}
override fun nextString(): String {
readNext()
return (stack.lastOrNull() ?: endReader).nextString()
}
override fun nextBoolean(): Boolean {
readNext()
return (stack.lastOrNull() ?: endReader).nextBoolean()
}
override fun skipValue() {
readNext()
return (stack.lastOrNull() ?: endReader).skipValue()
}
companion object {
const val TYPE_NULL = 0x01
const val TYPE_DOUBLE = 0x02
const val TYPE_BOOLEAN = 0x03
/**
* На самом деле, variable int
*/
const val TYPE_INT = 0x04
const val TYPE_STRING = 0x05
const val TYPE_ARRAY = 0x06
const val TYPE_OBJECT = 0x07
private fun fixSignedInt(read: Long): Long {
val sign = read and 0x1L
@Suppress("name_shadowing")
val read = read ushr 1
if (sign == 1L) {
return -read - 1L
} else {
return read
}
}
/**
* Позволяет читать двоичный JSON прямиком в [JsonElement]
*/
fun readElement(reader: RandomAccessFile): JsonElement {
return when (val id = reader.read()) {
TYPE_NULL -> JsonNull.INSTANCE
TYPE_DOUBLE -> JsonPrimitive(reader.readDouble())
TYPE_BOOLEAN -> JsonPrimitive(reader.readBoolean())
TYPE_INT -> JsonPrimitive(fixSignedInt(reader.readVarLong()))
TYPE_STRING -> JsonPrimitive(reader.readASCIIString(reader.readVarInt()))
TYPE_ARRAY -> readArray(reader)
TYPE_OBJECT -> readObject(reader)
else -> throw JsonParseException("Unknown element type $id")
}
}
/**
* Позволяет читать двоичный JSON прямиком в [JsonElement]
*/
fun readElement(reader: DataInputStream): JsonElement {
return when (val id = reader.read()) {
TYPE_NULL -> JsonNull.INSTANCE
TYPE_DOUBLE -> JsonPrimitive(reader.readDouble())
TYPE_BOOLEAN -> JsonPrimitive(reader.readBoolean())
TYPE_INT -> JsonPrimitive(fixSignedInt(reader.readVarLong()))
TYPE_STRING -> JsonPrimitive(reader.readASCIIString(reader.readVarInt()))
TYPE_ARRAY -> readArray(reader)
TYPE_OBJECT -> readObject(reader)
else -> throw JsonParseException("Unknown element type $id")
}
}
/**
* Позволяет читать двоичный JSON объект прямиком в [JsonObject]
*/
fun readObject(reader: RandomAccessFile): JsonObject {
val values = reader.readVarInt() - 1
if (values == -1) return JsonObject()
if (values < -1) throw JsonSyntaxException("Tried to read json object with $values elements in it")
val build = JsonObject()
for (i in 0 .. values) {
val key: String
try {
key = reader.readASCIIString(reader.readVarInt())
} catch(err: Throwable) {
throw JsonSyntaxException("Reading json object at $i", err)
}
try {
build.add(key, readElement(reader))
} catch(err: Throwable) {
throw JsonSyntaxException("Reading json object at $i with name $key", err)
}
}
return build
}
/**
* Позволяет читать двоичный JSON объект прямиком в [JsonObject]
*/
fun readObject(reader: DataInputStream): JsonObject {
val values = reader.readVarInt() - 1
if (values == -1) return JsonObject()
if (values < -1) throw JsonSyntaxException("Tried to read json object with $values elements in it")
val build = JsonObject()
for (i in 0 .. values) {
val key: String
try {
key = reader.readASCIIString(reader.readVarInt())
} catch(err: Throwable) {
throw JsonSyntaxException("Reading json object at $i", err)
}
try {
build.add(key, readElement(reader))
} catch(err: Throwable) {
throw JsonSyntaxException("Reading json object at $i with name $key", err)
}
}
return build
}
/**
* Позволяет читать двоичный JSON массив прямиком в [JsonArray]
*/
fun readArray(reader: RandomAccessFile): JsonArray {
val values = reader.readVarInt() - 1
if (values == -1) return JsonArray()
if (values < -1) throw JsonSyntaxException("Tried to read json array with $values elements in it")
val build = JsonArray(values)
for (i in 0 .. values) build.add(readElement(reader))
return build
}
/**
* Позволяет читать двоичный JSON массив прямиком в [JsonArray]
*/
fun readArray(reader: DataInputStream): JsonArray {
val values = reader.readVarInt() - 1
if (values == -1) return JsonArray()
if (values < -1) throw JsonSyntaxException("Tried to read json array with $values elements in it")
val build = JsonArray(values)
for (i in 0 .. values) build.add(readElement(reader))
return build
}
private val unreadable = object : Reader() {
override fun read(cbuf: CharArray, off: Int, len: Int): Int {
throw AssertionError()
}
override fun close() {
throw AssertionError()
}
}
}
}