DBot/KStarbound
DBot/KStarbound
3
0
Fork 0
Code Issues 1 Pull Requests Packages Projects Releases Wiki Activity

Initial implementation of zstd instead of zlib in storage

Browse Source
This commit is contained in:
DBotThePony 2024-12-09 00:12:34 +07:00
parent de34b8ac5f
commit c5fa4e4b59
Signed by: DBot
GPG Key ID: DCC23B5715498507
5 changed files with 118 additions and 55 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
build.gradle.kts
View File

@ -22,6 +22,7 @@ val guavaVersion: String by project
val junitVersion: String by project
val sqliteVersion: String by project
val picocliVersion: String by project
val zstdVersion: String by project
repositories {
maven {
@ -59,6 +60,7 @@ dependencies {
testRuntimeOnly("org.junit.jupiter:junit-jupiter-engine")
implementation("com.google.code.gson:gson:$gsonVersion")
implementation("com.github.luben:zstd-jni:$zstdVersion")
implementation("it.unimi.dsi:fastutil:$fastutilVersion")

1
gradle.properties
View File

@ -17,5 +17,6 @@ log4jVersion=2.17.1
guavaVersion=33.0.0-jre
junitVersion=5.8.2
sqliteVersion=3.45.3.0
zstdVersion=1.5.6-8
picocliVersion=4.7.6

59
src/main/kotlin/ru/dbotthepony/kstarbound/json/BinaryJsonReader.kt
View File

@ -1,5 +1,6 @@
package ru.dbotthepony.kstarbound.json
import com.github.luben.zstd.ZstdInputStreamNoFinalizer
import com.google.gson.JsonArray
import com.google.gson.JsonElement
import com.google.gson.JsonNull
@ -29,31 +30,55 @@ import java.util.zip.DeflaterOutputStream
import java.util.zip.Inflater
import java.util.zip.InflaterInputStream
private fun <T> ByteArray.callRead(inflate: Boolean, callable: DataInputStream.() -> T): T {
private enum class InflateType {
ZLIB,
ZSTD,
NONE
}
private fun <T> ByteArray.callRead(inflate: InflateType, callable: DataInputStream.() -> T): T {
val stream = FastByteArrayInputStream(this)
if (inflate) {
val inflater = Inflater()
when (inflate) {
InflateType.ZLIB -> {
val inflater = Inflater()
try {
val data = DataInputStream(BufferedInputStream(InflaterInputStream(stream, inflater, 0x4000), 0x10000))
val t = callable(data)
return t
} finally {
inflater.end()
try {
val data = DataInputStream(BufferedInputStream(InflaterInputStream(stream, inflater, 0x4000), 0x10000))
val t = callable(data)
return t
} finally {
inflater.end()
}
}
InflateType.ZSTD -> {
val data = DataInputStream(BufferedInputStream(ZstdInputStreamNoFinalizer(stream), 0x10000))
try {
return callable(data)
} finally {
data.close()
}
}
InflateType.NONE -> {
return callable(DataInputStream(stream))
}
} else {
return callable(DataInputStream(stream))
}
}
fun ByteArray.readJsonElement(): JsonElement = callRead(false) { readJsonElement() }
fun ByteArray.readJsonObject(): JsonObject = callRead(false) { readJsonObject() }
fun ByteArray.readJsonArray(): JsonArray = callRead(false) { readJsonArray() }
fun ByteArray.readJsonElement(): JsonElement = callRead(InflateType.NONE) { readJsonElement() }
fun ByteArray.readJsonObject(): JsonObject = callRead(InflateType.NONE) { readJsonObject() }
fun ByteArray.readJsonArray(): JsonArray = callRead(InflateType.NONE) { readJsonArray() }
fun ByteArray.readJsonElementInflated(): JsonElement = callRead(true) { readJsonElement() }
fun ByteArray.readJsonObjectInflated(): JsonObject = callRead(true) { readJsonObject() }
fun ByteArray.readJsonArrayInflated(): JsonArray = callRead(true) { readJsonArray() }
fun ByteArray.readJsonElementInflated(): JsonElement = callRead(InflateType.ZLIB) { readJsonElement() }
fun ByteArray.readJsonObjectInflated(): JsonObject = callRead(InflateType.ZLIB) { readJsonObject() }
fun ByteArray.readJsonArrayInflated(): JsonArray = callRead(InflateType.ZLIB) { readJsonArray() }
fun ByteArray.readJsonElementZstd(): JsonElement = callRead(InflateType.ZSTD) { readJsonElement() }
fun ByteArray.readJsonObjectZstd(): JsonObject = callRead(InflateType.ZSTD) { readJsonObject() }
fun ByteArray.readJsonArrayZstd(): JsonArray = callRead(InflateType.ZSTD) { readJsonArray() }
/**
* Позволяет читать двоичный JSON прямиком в [JsonElement]

49
src/main/kotlin/ru/dbotthepony/kstarbound/json/BinaryJsonWriter.kt
View File

@ -1,5 +1,6 @@
package ru.dbotthepony.kstarbound.json
import com.github.luben.zstd.ZstdOutputStreamNoFinalizer
import com.google.gson.JsonArray
import com.google.gson.JsonElement
import com.google.gson.JsonNull
@ -14,31 +15,51 @@ import java.io.DataOutputStream
import java.util.zip.Deflater
import java.util.zip.DeflaterOutputStream
private fun <T> T.callWrite(deflate: Boolean, callable: DataOutputStream.(T) -> Unit): ByteArray {
private enum class DeflateType {
ZLIB,
ZSTD,
NONE
}
private fun <T> T.callWrite(deflate: DeflateType, callable: DataOutputStream.(T) -> Unit): ByteArray {
val stream = FastByteArrayOutputStream()
if (deflate) {
val deflater = Deflater()
val data = DataOutputStream(BufferedOutputStream(DeflaterOutputStream(stream, deflater, 0x4000), 0x10000))
when (deflate) {
DeflateType.ZLIB -> {
val deflater = Deflater()
val data = DataOutputStream(BufferedOutputStream(DeflaterOutputStream(stream, deflater, 0x4000), 0x10000))
data.use {
callable(data, this)
data.use {
callable(data, this)
}
}
} else {
callable(DataOutputStream(stream), this)
DeflateType.ZSTD -> {
val s = ZstdOutputStreamNoFinalizer(stream)
s.setLevel(6)
DataOutputStream(BufferedOutputStream(s, 0x10000)).use {
callable(it, this)
}
}
DeflateType.NONE -> callable(DataOutputStream(stream), this)
}
return stream.array.copyOf(stream.length)
}
fun JsonElement.writeJsonElement(): ByteArray = callWrite(false) { writeJsonElement(it) }
fun JsonObject.writeJsonObject(): ByteArray = callWrite(false) { writeJsonObject(it) }
fun JsonArray.writeJsonArray(): ByteArray = callWrite(false) { writeJsonArray(it) }
fun JsonElement.writeJsonElement(): ByteArray = callWrite(DeflateType.NONE) { writeJsonElement(it) }
fun JsonObject.writeJsonObject(): ByteArray = callWrite(DeflateType.NONE) { writeJsonObject(it) }
fun JsonArray.writeJsonArray(): ByteArray = callWrite(DeflateType.NONE) { writeJsonArray(it) }
fun JsonElement.writeJsonElementDeflated(): ByteArray = callWrite(true) { writeJsonElement(it) }
fun JsonObject.writeJsonObjectDeflated(): ByteArray = callWrite(true) { writeJsonObject(it) }
fun JsonArray.writeJsonArrayDeflated(): ByteArray = callWrite(true) { writeJsonArray(it) }
fun JsonElement.writeJsonElementDeflated(): ByteArray = callWrite(DeflateType.ZLIB) { writeJsonElement(it) }
fun JsonObject.writeJsonObjectDeflated(): ByteArray = callWrite(DeflateType.ZLIB) { writeJsonObject(it) }
fun JsonArray.writeJsonArrayDeflated(): ByteArray = callWrite(DeflateType.ZLIB) { writeJsonArray(it) }
fun JsonElement.writeJsonElementZstd(): ByteArray = callWrite(DeflateType.ZSTD) { writeJsonElement(it) }
fun JsonObject.writeJsonObjectZstd(): ByteArray = callWrite(DeflateType.ZSTD) { writeJsonObject(it) }
fun JsonArray.writeJsonArrayZstd(): ByteArray = callWrite(DeflateType.ZSTD) { writeJsonArray(it) }
fun DataOutputStream.writeJsonElement(value: JsonElement) {
when (value) {

62
src/main/kotlin/ru/dbotthepony/kstarbound/server/world/NativeLocalWorldStorage.kt
View File

@ -1,5 +1,7 @@
package ru.dbotthepony.kstarbound.server.world
import com.github.luben.zstd.ZstdInputStreamNoFinalizer
import com.github.luben.zstd.ZstdOutputStreamNoFinalizer
import com.google.gson.JsonArray
import com.google.gson.JsonObject
import it.unimi.dsi.fastutil.ints.Int2ObjectAVLTreeMap
@ -19,9 +21,11 @@ import ru.dbotthepony.kstarbound.defs.EntityType
import ru.dbotthepony.kstarbound.io.SQLSavepoint
import ru.dbotthepony.kstarbound.json.VersionedJson
import ru.dbotthepony.kstarbound.json.readJsonArrayInflated
import ru.dbotthepony.kstarbound.json.readJsonArrayZstd
import ru.dbotthepony.kstarbound.json.readJsonElement
import ru.dbotthepony.kstarbound.json.readJsonObjectInflated
import ru.dbotthepony.kstarbound.json.writeJsonArrayDeflated
import ru.dbotthepony.kstarbound.json.writeJsonArrayZstd
import ru.dbotthepony.kstarbound.json.writeJsonElement
import ru.dbotthepony.kstarbound.math.vector.Vector2d
import ru.dbotthepony.kstarbound.math.vector.Vector2i
@ -140,8 +144,7 @@ class NativeLocalWorldStorage(file: File?) : WorldStorage() {
val version = it.getInt(2)
val data = it.getBytes(3)
val inflater = Inflater()
val stream = DataInputStream(BufferedInputStream(InflaterInputStream(FastByteArrayInputStream(data), inflater, 0x10000), 0x40000))
val stream = DataInputStream(BufferedInputStream(ZstdInputStreamNoFinalizer(FastByteArrayInputStream(data)), 0x40000))
try {
val palette = PaletteSet()
@ -159,7 +162,7 @@ class NativeLocalWorldStorage(file: File?) : WorldStorage() {
return@supplyAsync ChunkCells(array as Object2DArray<out AbstractCell>, stage)
} finally {
inflater.end()
stream.close()
}
}
}
@ -178,7 +181,7 @@ class NativeLocalWorldStorage(file: File?) : WorldStorage() {
readEntities.executeQuery().use {
if (it.next()) {
val data = it.getBytes(1).readJsonArrayInflated()
val data = it.getBytes(1).readJsonArrayZstd()
for (entry in data) {
val versioned = VersionedJson.fromJson(entry)
@ -228,16 +231,19 @@ class NativeLocalWorldStorage(file: File?) : WorldStorage() {
return executor.supplyAsync {
val (version, metadata) = readMetadata("metadata") ?: return@supplyAsync null
val stream = DataInputStream(BufferedInputStream(InflaterInputStream(FastByteArrayInputStream(metadata))))
val stream = DataInputStream(BufferedInputStream(ZstdInputStreamNoFinalizer(FastByteArrayInputStream(metadata))))
val width = stream.readInt()
val height = stream.readInt()
val loopX = stream.readBoolean()
val loopY = stream.readBoolean()
val json = VersionedJson("WorldMetadata", version, stream.readJsonElement())
try {
val width = stream.readInt()
val height = stream.readInt()
val loopX = stream.readBoolean()
val loopY = stream.readBoolean()
val json = VersionedJson("WorldMetadata", version, stream.readJsonElement())
stream.close()
Metadata(WorldGeometry(Vector2i(width, height), loopX, loopY), json)
Metadata(WorldGeometry(Vector2i(width, height), loopX, loopY), json)
} finally {
stream.close()
}
}
}
@ -286,7 +292,7 @@ class NativeLocalWorldStorage(file: File?) : WorldStorage() {
writeEntities.setInt(1, pos.x)
writeEntities.setInt(2, pos.y)
writeEntities.setBytes(3, storeData.writeJsonArrayDeflated())
writeEntities.setBytes(3, storeData.writeJsonArrayZstd())
writeEntities.execute()
}
}
@ -380,9 +386,10 @@ class NativeLocalWorldStorage(file: File?) : WorldStorage() {
}
}
val deflater = Deflater()
val buff = FastByteArrayOutputStream()
val stream = DataOutputStream(BufferedOutputStream(DeflaterOutputStream(buff, deflater, 0x10000), 0x40000))
val z = ZstdOutputStreamNoFinalizer(buff)
z.setLevel(6)
val stream = DataOutputStream(BufferedOutputStream(z, 0x40000))
try {
palette.write(stream)
@ -405,7 +412,7 @@ class NativeLocalWorldStorage(file: File?) : WorldStorage() {
writeCells.setBytes(5, buff.array.copyOf(buff.length))
writeCells.execute()
} finally {
deflater.end()
z.close()
}
}
}
@ -413,16 +420,23 @@ class NativeLocalWorldStorage(file: File?) : WorldStorage() {
override fun saveMetadata(data: Metadata) {
executor.execute {
val buff = FastByteArrayOutputStream()
val stream = DataOutputStream(BufferedOutputStream(DeflaterOutputStream(buff, Deflater(), 0x10000), 0x40000))
val z = ZstdOutputStreamNoFinalizer(buff)
z.setLevel(6)
stream.writeInt(data.geometry.size.x)
stream.writeInt(data.geometry.size.y)
stream.writeBoolean(data.geometry.loopX)
stream.writeBoolean(data.geometry.loopY)
stream.writeJsonElement(data.data.content)
try {
val stream = DataOutputStream(BufferedOutputStream(z, 0x40000))
stream.close()
writeMetadata("metadata", data.data.version ?: 0, buff.array.copyOf(buff.length))
stream.writeInt(data.geometry.size.x)
stream.writeInt(data.geometry.size.y)
stream.writeBoolean(data.geometry.loopX)
stream.writeBoolean(data.geometry.loopY)
stream.writeJsonElement(data.data.content)
stream.close()
writeMetadata("metadata", data.data.version ?: 0, buff.array.copyOf(buff.length))
} finally {
z.close()
}
}
}