diff --git a/build.gradle.kts b/build.gradle.kts index 215558a9..f7e14840 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -22,6 +22,7 @@ val guavaVersion: String by project val junitVersion: String by project val sqliteVersion: String by project val picocliVersion: String by project +val zstdVersion: String by project repositories { maven { @@ -59,6 +60,7 @@ dependencies { testRuntimeOnly("org.junit.jupiter:junit-jupiter-engine") implementation("com.google.code.gson:gson:$gsonVersion") + implementation("com.github.luben:zstd-jni:$zstdVersion") implementation("it.unimi.dsi:fastutil:$fastutilVersion") diff --git a/gradle.properties b/gradle.properties index ad06b2dc..c89f7f30 100644 --- a/gradle.properties +++ b/gradle.properties @@ -17,5 +17,6 @@ log4jVersion=2.17.1 guavaVersion=33.0.0-jre junitVersion=5.8.2 sqliteVersion=3.45.3.0 +zstdVersion=1.5.6-8 picocliVersion=4.7.6 diff --git a/src/main/kotlin/ru/dbotthepony/kstarbound/json/BinaryJsonReader.kt b/src/main/kotlin/ru/dbotthepony/kstarbound/json/BinaryJsonReader.kt index af66ad38..f9a7de15 100644 --- a/src/main/kotlin/ru/dbotthepony/kstarbound/json/BinaryJsonReader.kt +++ b/src/main/kotlin/ru/dbotthepony/kstarbound/json/BinaryJsonReader.kt @@ -1,5 +1,6 @@ package ru.dbotthepony.kstarbound.json +import com.github.luben.zstd.ZstdInputStreamNoFinalizer import com.google.gson.JsonArray import com.google.gson.JsonElement import com.google.gson.JsonNull @@ -29,31 +30,55 @@ import java.util.zip.DeflaterOutputStream import java.util.zip.Inflater import java.util.zip.InflaterInputStream -private fun ByteArray.callRead(inflate: Boolean, callable: DataInputStream.() -> T): T { +private enum class InflateType { + ZLIB, + ZSTD, + NONE +} + +private fun ByteArray.callRead(inflate: InflateType, callable: DataInputStream.() -> T): T { val stream = FastByteArrayInputStream(this) - if (inflate) { - val inflater = Inflater() + when (inflate) { + InflateType.ZLIB -> { + val inflater = Inflater() - try { - val data = DataInputStream(BufferedInputStream(InflaterInputStream(stream, inflater, 0x4000), 0x10000)) - val t = callable(data) - return t - } finally { - inflater.end() + try { + val data = DataInputStream(BufferedInputStream(InflaterInputStream(stream, inflater, 0x4000), 0x10000)) + val t = callable(data) + return t + } finally { + inflater.end() + } + } + + InflateType.ZSTD -> { + val data = DataInputStream(BufferedInputStream(ZstdInputStreamNoFinalizer(stream), 0x10000)) + + try { + return callable(data) + } finally { + data.close() + } + } + + InflateType.NONE -> { + return callable(DataInputStream(stream)) } - } else { - return callable(DataInputStream(stream)) } } -fun ByteArray.readJsonElement(): JsonElement = callRead(false) { readJsonElement() } -fun ByteArray.readJsonObject(): JsonObject = callRead(false) { readJsonObject() } -fun ByteArray.readJsonArray(): JsonArray = callRead(false) { readJsonArray() } +fun ByteArray.readJsonElement(): JsonElement = callRead(InflateType.NONE) { readJsonElement() } +fun ByteArray.readJsonObject(): JsonObject = callRead(InflateType.NONE) { readJsonObject() } +fun ByteArray.readJsonArray(): JsonArray = callRead(InflateType.NONE) { readJsonArray() } -fun ByteArray.readJsonElementInflated(): JsonElement = callRead(true) { readJsonElement() } -fun ByteArray.readJsonObjectInflated(): JsonObject = callRead(true) { readJsonObject() } -fun ByteArray.readJsonArrayInflated(): JsonArray = callRead(true) { readJsonArray() } +fun ByteArray.readJsonElementInflated(): JsonElement = callRead(InflateType.ZLIB) { readJsonElement() } +fun ByteArray.readJsonObjectInflated(): JsonObject = callRead(InflateType.ZLIB) { readJsonObject() } +fun ByteArray.readJsonArrayInflated(): JsonArray = callRead(InflateType.ZLIB) { readJsonArray() } + +fun ByteArray.readJsonElementZstd(): JsonElement = callRead(InflateType.ZSTD) { readJsonElement() } +fun ByteArray.readJsonObjectZstd(): JsonObject = callRead(InflateType.ZSTD) { readJsonObject() } +fun ByteArray.readJsonArrayZstd(): JsonArray = callRead(InflateType.ZSTD) { readJsonArray() } /** * Позволяет читать двоичный JSON прямиком в [JsonElement] diff --git a/src/main/kotlin/ru/dbotthepony/kstarbound/json/BinaryJsonWriter.kt b/src/main/kotlin/ru/dbotthepony/kstarbound/json/BinaryJsonWriter.kt index ae689ed4..bafc753a 100644 --- a/src/main/kotlin/ru/dbotthepony/kstarbound/json/BinaryJsonWriter.kt +++ b/src/main/kotlin/ru/dbotthepony/kstarbound/json/BinaryJsonWriter.kt @@ -1,5 +1,6 @@ package ru.dbotthepony.kstarbound.json +import com.github.luben.zstd.ZstdOutputStreamNoFinalizer import com.google.gson.JsonArray import com.google.gson.JsonElement import com.google.gson.JsonNull @@ -14,31 +15,51 @@ import java.io.DataOutputStream import java.util.zip.Deflater import java.util.zip.DeflaterOutputStream -private fun T.callWrite(deflate: Boolean, callable: DataOutputStream.(T) -> Unit): ByteArray { +private enum class DeflateType { + ZLIB, + ZSTD, + NONE +} + +private fun T.callWrite(deflate: DeflateType, callable: DataOutputStream.(T) -> Unit): ByteArray { val stream = FastByteArrayOutputStream() - if (deflate) { - val deflater = Deflater() - val data = DataOutputStream(BufferedOutputStream(DeflaterOutputStream(stream, deflater, 0x4000), 0x10000)) + when (deflate) { + DeflateType.ZLIB -> { + val deflater = Deflater() + val data = DataOutputStream(BufferedOutputStream(DeflaterOutputStream(stream, deflater, 0x4000), 0x10000)) - data.use { - callable(data, this) + data.use { + callable(data, this) + } } - } else { - callable(DataOutputStream(stream), this) + DeflateType.ZSTD -> { + val s = ZstdOutputStreamNoFinalizer(stream) + s.setLevel(6) + + DataOutputStream(BufferedOutputStream(s, 0x10000)).use { + callable(it, this) + } + } + + DeflateType.NONE -> callable(DataOutputStream(stream), this) } return stream.array.copyOf(stream.length) } -fun JsonElement.writeJsonElement(): ByteArray = callWrite(false) { writeJsonElement(it) } -fun JsonObject.writeJsonObject(): ByteArray = callWrite(false) { writeJsonObject(it) } -fun JsonArray.writeJsonArray(): ByteArray = callWrite(false) { writeJsonArray(it) } +fun JsonElement.writeJsonElement(): ByteArray = callWrite(DeflateType.NONE) { writeJsonElement(it) } +fun JsonObject.writeJsonObject(): ByteArray = callWrite(DeflateType.NONE) { writeJsonObject(it) } +fun JsonArray.writeJsonArray(): ByteArray = callWrite(DeflateType.NONE) { writeJsonArray(it) } -fun JsonElement.writeJsonElementDeflated(): ByteArray = callWrite(true) { writeJsonElement(it) } -fun JsonObject.writeJsonObjectDeflated(): ByteArray = callWrite(true) { writeJsonObject(it) } -fun JsonArray.writeJsonArrayDeflated(): ByteArray = callWrite(true) { writeJsonArray(it) } +fun JsonElement.writeJsonElementDeflated(): ByteArray = callWrite(DeflateType.ZLIB) { writeJsonElement(it) } +fun JsonObject.writeJsonObjectDeflated(): ByteArray = callWrite(DeflateType.ZLIB) { writeJsonObject(it) } +fun JsonArray.writeJsonArrayDeflated(): ByteArray = callWrite(DeflateType.ZLIB) { writeJsonArray(it) } + +fun JsonElement.writeJsonElementZstd(): ByteArray = callWrite(DeflateType.ZSTD) { writeJsonElement(it) } +fun JsonObject.writeJsonObjectZstd(): ByteArray = callWrite(DeflateType.ZSTD) { writeJsonObject(it) } +fun JsonArray.writeJsonArrayZstd(): ByteArray = callWrite(DeflateType.ZSTD) { writeJsonArray(it) } fun DataOutputStream.writeJsonElement(value: JsonElement) { when (value) { diff --git a/src/main/kotlin/ru/dbotthepony/kstarbound/server/world/NativeLocalWorldStorage.kt b/src/main/kotlin/ru/dbotthepony/kstarbound/server/world/NativeLocalWorldStorage.kt index a4624021..bbab3612 100644 --- a/src/main/kotlin/ru/dbotthepony/kstarbound/server/world/NativeLocalWorldStorage.kt +++ b/src/main/kotlin/ru/dbotthepony/kstarbound/server/world/NativeLocalWorldStorage.kt @@ -1,5 +1,7 @@ package ru.dbotthepony.kstarbound.server.world +import com.github.luben.zstd.ZstdInputStreamNoFinalizer +import com.github.luben.zstd.ZstdOutputStreamNoFinalizer import com.google.gson.JsonArray import com.google.gson.JsonObject import it.unimi.dsi.fastutil.ints.Int2ObjectAVLTreeMap @@ -19,9 +21,11 @@ import ru.dbotthepony.kstarbound.defs.EntityType import ru.dbotthepony.kstarbound.io.SQLSavepoint import ru.dbotthepony.kstarbound.json.VersionedJson import ru.dbotthepony.kstarbound.json.readJsonArrayInflated +import ru.dbotthepony.kstarbound.json.readJsonArrayZstd import ru.dbotthepony.kstarbound.json.readJsonElement import ru.dbotthepony.kstarbound.json.readJsonObjectInflated import ru.dbotthepony.kstarbound.json.writeJsonArrayDeflated +import ru.dbotthepony.kstarbound.json.writeJsonArrayZstd import ru.dbotthepony.kstarbound.json.writeJsonElement import ru.dbotthepony.kstarbound.math.vector.Vector2d import ru.dbotthepony.kstarbound.math.vector.Vector2i @@ -140,8 +144,7 @@ class NativeLocalWorldStorage(file: File?) : WorldStorage() { val version = it.getInt(2) val data = it.getBytes(3) - val inflater = Inflater() - val stream = DataInputStream(BufferedInputStream(InflaterInputStream(FastByteArrayInputStream(data), inflater, 0x10000), 0x40000)) + val stream = DataInputStream(BufferedInputStream(ZstdInputStreamNoFinalizer(FastByteArrayInputStream(data)), 0x40000)) try { val palette = PaletteSet() @@ -159,7 +162,7 @@ class NativeLocalWorldStorage(file: File?) : WorldStorage() { return@supplyAsync ChunkCells(array as Object2DArray, stage) } finally { - inflater.end() + stream.close() } } } @@ -178,7 +181,7 @@ class NativeLocalWorldStorage(file: File?) : WorldStorage() { readEntities.executeQuery().use { if (it.next()) { - val data = it.getBytes(1).readJsonArrayInflated() + val data = it.getBytes(1).readJsonArrayZstd() for (entry in data) { val versioned = VersionedJson.fromJson(entry) @@ -228,16 +231,19 @@ class NativeLocalWorldStorage(file: File?) : WorldStorage() { return executor.supplyAsync { val (version, metadata) = readMetadata("metadata") ?: return@supplyAsync null - val stream = DataInputStream(BufferedInputStream(InflaterInputStream(FastByteArrayInputStream(metadata)))) + val stream = DataInputStream(BufferedInputStream(ZstdInputStreamNoFinalizer(FastByteArrayInputStream(metadata)))) - val width = stream.readInt() - val height = stream.readInt() - val loopX = stream.readBoolean() - val loopY = stream.readBoolean() - val json = VersionedJson("WorldMetadata", version, stream.readJsonElement()) + try { + val width = stream.readInt() + val height = stream.readInt() + val loopX = stream.readBoolean() + val loopY = stream.readBoolean() + val json = VersionedJson("WorldMetadata", version, stream.readJsonElement()) - stream.close() - Metadata(WorldGeometry(Vector2i(width, height), loopX, loopY), json) + Metadata(WorldGeometry(Vector2i(width, height), loopX, loopY), json) + } finally { + stream.close() + } } } @@ -286,7 +292,7 @@ class NativeLocalWorldStorage(file: File?) : WorldStorage() { writeEntities.setInt(1, pos.x) writeEntities.setInt(2, pos.y) - writeEntities.setBytes(3, storeData.writeJsonArrayDeflated()) + writeEntities.setBytes(3, storeData.writeJsonArrayZstd()) writeEntities.execute() } } @@ -380,9 +386,10 @@ class NativeLocalWorldStorage(file: File?) : WorldStorage() { } } - val deflater = Deflater() val buff = FastByteArrayOutputStream() - val stream = DataOutputStream(BufferedOutputStream(DeflaterOutputStream(buff, deflater, 0x10000), 0x40000)) + val z = ZstdOutputStreamNoFinalizer(buff) + z.setLevel(6) + val stream = DataOutputStream(BufferedOutputStream(z, 0x40000)) try { palette.write(stream) @@ -405,7 +412,7 @@ class NativeLocalWorldStorage(file: File?) : WorldStorage() { writeCells.setBytes(5, buff.array.copyOf(buff.length)) writeCells.execute() } finally { - deflater.end() + z.close() } } } @@ -413,16 +420,23 @@ class NativeLocalWorldStorage(file: File?) : WorldStorage() { override fun saveMetadata(data: Metadata) { executor.execute { val buff = FastByteArrayOutputStream() - val stream = DataOutputStream(BufferedOutputStream(DeflaterOutputStream(buff, Deflater(), 0x10000), 0x40000)) + val z = ZstdOutputStreamNoFinalizer(buff) + z.setLevel(6) - stream.writeInt(data.geometry.size.x) - stream.writeInt(data.geometry.size.y) - stream.writeBoolean(data.geometry.loopX) - stream.writeBoolean(data.geometry.loopY) - stream.writeJsonElement(data.data.content) + try { + val stream = DataOutputStream(BufferedOutputStream(z, 0x40000)) - stream.close() - writeMetadata("metadata", data.data.version ?: 0, buff.array.copyOf(buff.length)) + stream.writeInt(data.geometry.size.x) + stream.writeInt(data.geometry.size.y) + stream.writeBoolean(data.geometry.loopX) + stream.writeBoolean(data.geometry.loopY) + stream.writeJsonElement(data.data.content) + + stream.close() + writeMetadata("metadata", data.data.version ?: 0, buff.array.copyOf(buff.length)) + } finally { + z.close() + } } }