Initial implementation of zstd instead of zlib in storage

build.gradle.kts

@@ -22,6 +22,7 @@ val guavaVersion: String by project
 val junitVersion: String by project
 val sqliteVersion: String by project
 val picocliVersion: String by project
+val zstdVersion: String by project
 
 repositories {
 	maven {
@@ -59,6 +60,7 @@ dependencies {
 	testRuntimeOnly("org.junit.jupiter:junit-jupiter-engine")
 
 	implementation("com.google.code.gson:gson:$gsonVersion")
+	implementation("com.github.luben:zstd-jni:$zstdVersion")
 
 	implementation("it.unimi.dsi:fastutil:$fastutilVersion")
 

gradle.properties

@@ -17,5 +17,6 @@ log4jVersion=2.17.1
 guavaVersion=33.0.0-jre
 junitVersion=5.8.2
 sqliteVersion=3.45.3.0
+zstdVersion=1.5.6-8
 
 picocliVersion=4.7.6

src/main/kotlin/ru/dbotthepony/kstarbound/json/BinaryJsonReader.kt

@@ -1,5 +1,6 @@
 package ru.dbotthepony.kstarbound.json
 
+import com.github.luben.zstd.ZstdInputStreamNoFinalizer
 import com.google.gson.JsonArray
 import com.google.gson.JsonElement
 import com.google.gson.JsonNull
@@ -29,31 +30,55 @@ import java.util.zip.DeflaterOutputStream
 import java.util.zip.Inflater
 import java.util.zip.InflaterInputStream
 
-private fun <T> ByteArray.callRead(inflate: Boolean, callable: DataInputStream.() -> T): T {
+private enum class InflateType {
+	ZLIB,
+	ZSTD,
+	NONE
+}
+
+private fun <T> ByteArray.callRead(inflate: InflateType, callable: DataInputStream.() -> T): T {
 	val stream = FastByteArrayInputStream(this)
 
-	if (inflate) {
+	when (inflate) {
-		val inflater = Inflater()
+		InflateType.ZLIB -> {
+			val inflater = Inflater()
 
-		try {
+			try {
-			val data = DataInputStream(BufferedInputStream(InflaterInputStream(stream, inflater, 0x4000), 0x10000))
+				val data = DataInputStream(BufferedInputStream(InflaterInputStream(stream, inflater, 0x4000), 0x10000))
-			val t = callable(data)
+				val t = callable(data)
-			return t
+				return t
-		} finally {
+			} finally {
-			inflater.end()
+				inflater.end()
+			}
+		}
+
+		InflateType.ZSTD -> {
+			val data = DataInputStream(BufferedInputStream(ZstdInputStreamNoFinalizer(stream), 0x10000))
+
+			try {
+				return callable(data)
+			} finally {
+				data.close()
+			}
+		}
+
+		InflateType.NONE -> {
+			return callable(DataInputStream(stream))
 		}
-	} else {
-		return callable(DataInputStream(stream))
 	}
 }
 
-fun ByteArray.readJsonElement(): JsonElement = callRead(false) { readJsonElement() }
+fun ByteArray.readJsonElement(): JsonElement = callRead(InflateType.NONE) { readJsonElement() }
-fun ByteArray.readJsonObject(): JsonObject = callRead(false) { readJsonObject() }
+fun ByteArray.readJsonObject(): JsonObject = callRead(InflateType.NONE) { readJsonObject() }
-fun ByteArray.readJsonArray(): JsonArray = callRead(false) { readJsonArray() }
+fun ByteArray.readJsonArray(): JsonArray = callRead(InflateType.NONE) { readJsonArray() }
 
-fun ByteArray.readJsonElementInflated(): JsonElement = callRead(true) { readJsonElement() }
+fun ByteArray.readJsonElementInflated(): JsonElement = callRead(InflateType.ZLIB) { readJsonElement() }
-fun ByteArray.readJsonObjectInflated(): JsonObject = callRead(true) { readJsonObject() }
+fun ByteArray.readJsonObjectInflated(): JsonObject = callRead(InflateType.ZLIB) { readJsonObject() }
-fun ByteArray.readJsonArrayInflated(): JsonArray = callRead(true) { readJsonArray() }
+fun ByteArray.readJsonArrayInflated(): JsonArray = callRead(InflateType.ZLIB) { readJsonArray() }
+
+fun ByteArray.readJsonElementZstd(): JsonElement = callRead(InflateType.ZSTD) { readJsonElement() }
+fun ByteArray.readJsonObjectZstd(): JsonObject = callRead(InflateType.ZSTD) { readJsonObject() }
+fun ByteArray.readJsonArrayZstd(): JsonArray = callRead(InflateType.ZSTD) { readJsonArray() }
 
 /**
  * Позволяет читать двоичный JSON прямиком в [JsonElement]

src/main/kotlin/ru/dbotthepony/kstarbound/json/BinaryJsonWriter.kt

@@ -1,5 +1,6 @@
 package ru.dbotthepony.kstarbound.json
 
+import com.github.luben.zstd.ZstdOutputStreamNoFinalizer
 import com.google.gson.JsonArray
 import com.google.gson.JsonElement
 import com.google.gson.JsonNull
@@ -14,31 +15,51 @@ import java.io.DataOutputStream
 import java.util.zip.Deflater
 import java.util.zip.DeflaterOutputStream
 
-private fun <T> T.callWrite(deflate: Boolean, callable: DataOutputStream.(T) -> Unit): ByteArray {
+private enum class DeflateType {
+	ZLIB,
+	ZSTD,
+	NONE
+}
+
+private fun <T> T.callWrite(deflate: DeflateType, callable: DataOutputStream.(T) -> Unit): ByteArray {
 	val stream = FastByteArrayOutputStream()
 
-	if (deflate) {
+	when (deflate) {
-		val deflater = Deflater()
+		DeflateType.ZLIB -> {
-		val data = DataOutputStream(BufferedOutputStream(DeflaterOutputStream(stream, deflater, 0x4000), 0x10000))
+			val deflater = Deflater()
+			val data = DataOutputStream(BufferedOutputStream(DeflaterOutputStream(stream, deflater, 0x4000), 0x10000))
 
-		data.use {
+			data.use {
-			callable(data, this)
+				callable(data, this)
+			}
 		}
 
-	} else {
+		DeflateType.ZSTD -> {
-		callable(DataOutputStream(stream), this)
+			val s = ZstdOutputStreamNoFinalizer(stream)
+			s.setLevel(6)
+
+			DataOutputStream(BufferedOutputStream(s, 0x10000)).use {
+				callable(it, this)
+			}
+		}
+
+		DeflateType.NONE -> callable(DataOutputStream(stream), this)
 	}
 
 	return stream.array.copyOf(stream.length)
 }
 
-fun JsonElement.writeJsonElement(): ByteArray = callWrite(false) { writeJsonElement(it) }
+fun JsonElement.writeJsonElement(): ByteArray = callWrite(DeflateType.NONE) { writeJsonElement(it) }
-fun JsonObject.writeJsonObject(): ByteArray = callWrite(false) { writeJsonObject(it) }
+fun JsonObject.writeJsonObject(): ByteArray = callWrite(DeflateType.NONE) { writeJsonObject(it) }
-fun JsonArray.writeJsonArray(): ByteArray = callWrite(false) { writeJsonArray(it) }
+fun JsonArray.writeJsonArray(): ByteArray = callWrite(DeflateType.NONE) { writeJsonArray(it) }
 
-fun JsonElement.writeJsonElementDeflated(): ByteArray = callWrite(true) { writeJsonElement(it) }
+fun JsonElement.writeJsonElementDeflated(): ByteArray = callWrite(DeflateType.ZLIB) { writeJsonElement(it) }
-fun JsonObject.writeJsonObjectDeflated(): ByteArray = callWrite(true) { writeJsonObject(it) }
+fun JsonObject.writeJsonObjectDeflated(): ByteArray = callWrite(DeflateType.ZLIB) { writeJsonObject(it) }
-fun JsonArray.writeJsonArrayDeflated(): ByteArray = callWrite(true) { writeJsonArray(it) }
+fun JsonArray.writeJsonArrayDeflated(): ByteArray = callWrite(DeflateType.ZLIB) { writeJsonArray(it) }
+
+fun JsonElement.writeJsonElementZstd(): ByteArray = callWrite(DeflateType.ZSTD) { writeJsonElement(it) }
+fun JsonObject.writeJsonObjectZstd(): ByteArray = callWrite(DeflateType.ZSTD) { writeJsonObject(it) }
+fun JsonArray.writeJsonArrayZstd(): ByteArray = callWrite(DeflateType.ZSTD) { writeJsonArray(it) }
 
 fun DataOutputStream.writeJsonElement(value: JsonElement) {
 	when (value) {

src/main/kotlin/ru/dbotthepony/kstarbound/server/world/NativeLocalWorldStorage.kt

@@ -1,5 +1,7 @@
 package ru.dbotthepony.kstarbound.server.world
 
+import com.github.luben.zstd.ZstdInputStreamNoFinalizer
+import com.github.luben.zstd.ZstdOutputStreamNoFinalizer
 import com.google.gson.JsonArray
 import com.google.gson.JsonObject
 import it.unimi.dsi.fastutil.ints.Int2ObjectAVLTreeMap
@@ -19,9 +21,11 @@ import ru.dbotthepony.kstarbound.defs.EntityType
 import ru.dbotthepony.kstarbound.io.SQLSavepoint
 import ru.dbotthepony.kstarbound.json.VersionedJson
 import ru.dbotthepony.kstarbound.json.readJsonArrayInflated
+import ru.dbotthepony.kstarbound.json.readJsonArrayZstd
 import ru.dbotthepony.kstarbound.json.readJsonElement
 import ru.dbotthepony.kstarbound.json.readJsonObjectInflated
 import ru.dbotthepony.kstarbound.json.writeJsonArrayDeflated
+import ru.dbotthepony.kstarbound.json.writeJsonArrayZstd
 import ru.dbotthepony.kstarbound.json.writeJsonElement
 import ru.dbotthepony.kstarbound.math.vector.Vector2d
 import ru.dbotthepony.kstarbound.math.vector.Vector2i
@@ -140,8 +144,7 @@ class NativeLocalWorldStorage(file: File?) : WorldStorage() {
 				val version = it.getInt(2)
 				val data = it.getBytes(3)
 
-				val inflater = Inflater()
+				val stream = DataInputStream(BufferedInputStream(ZstdInputStreamNoFinalizer(FastByteArrayInputStream(data)), 0x40000))
-				val stream = DataInputStream(BufferedInputStream(InflaterInputStream(FastByteArrayInputStream(data), inflater, 0x10000), 0x40000))
 
 				try {
 					val palette = PaletteSet()
@@ -159,7 +162,7 @@ class NativeLocalWorldStorage(file: File?) : WorldStorage() {
 
 					return@supplyAsync ChunkCells(array as Object2DArray<out AbstractCell>, stage)
 				} finally {
-					inflater.end()
+					stream.close()
 				}
 			}
 		}
@@ -178,7 +181,7 @@ class NativeLocalWorldStorage(file: File?) : WorldStorage() {
 
 			readEntities.executeQuery().use {
 				if (it.next()) {
-					val data = it.getBytes(1).readJsonArrayInflated()
+					val data = it.getBytes(1).readJsonArrayZstd()
 
 					for (entry in data) {
 						val versioned = VersionedJson.fromJson(entry)
@@ -228,16 +231,19 @@ class NativeLocalWorldStorage(file: File?) : WorldStorage() {
 		return executor.supplyAsync {
 			val (version, metadata) = readMetadata("metadata") ?: return@supplyAsync null
 
-			val stream = DataInputStream(BufferedInputStream(InflaterInputStream(FastByteArrayInputStream(metadata))))
+			val stream = DataInputStream(BufferedInputStream(ZstdInputStreamNoFinalizer(FastByteArrayInputStream(metadata))))
 
-			val width = stream.readInt()
+			try {
-			val height = stream.readInt()
+				val width = stream.readInt()
-			val loopX = stream.readBoolean()
+				val height = stream.readInt()
-			val loopY = stream.readBoolean()
+				val loopX = stream.readBoolean()
-			val json = VersionedJson("WorldMetadata", version, stream.readJsonElement())
+				val loopY = stream.readBoolean()
+				val json = VersionedJson("WorldMetadata", version, stream.readJsonElement())
 
-			stream.close()
+				Metadata(WorldGeometry(Vector2i(width, height), loopX, loopY), json)
-			Metadata(WorldGeometry(Vector2i(width, height), loopX, loopY), json)
+			} finally {
+				stream.close()
+			}
 		}
 	}
 
@@ -286,7 +292,7 @@ class NativeLocalWorldStorage(file: File?) : WorldStorage() {
 
 				writeEntities.setInt(1, pos.x)
 				writeEntities.setInt(2, pos.y)
-				writeEntities.setBytes(3, storeData.writeJsonArrayDeflated())
+				writeEntities.setBytes(3, storeData.writeJsonArrayZstd())
 				writeEntities.execute()
 			}
 		}
@@ -380,9 +386,10 @@ class NativeLocalWorldStorage(file: File?) : WorldStorage() {
 				}
 			}
 
-			val deflater = Deflater()
 			val buff = FastByteArrayOutputStream()
-			val stream = DataOutputStream(BufferedOutputStream(DeflaterOutputStream(buff, deflater, 0x10000), 0x40000))
+			val z = ZstdOutputStreamNoFinalizer(buff)
+			z.setLevel(6)
+			val stream = DataOutputStream(BufferedOutputStream(z, 0x40000))
 
 			try {
 				palette.write(stream)
@@ -405,7 +412,7 @@ class NativeLocalWorldStorage(file: File?) : WorldStorage() {
 				writeCells.setBytes(5, buff.array.copyOf(buff.length))
 				writeCells.execute()
 			} finally {
-				deflater.end()
+				z.close()
 			}
 		}
 	}
@@ -413,16 +420,23 @@ class NativeLocalWorldStorage(file: File?) : WorldStorage() {
 	override fun saveMetadata(data: Metadata) {
 		executor.execute {
 			val buff = FastByteArrayOutputStream()
-			val stream = DataOutputStream(BufferedOutputStream(DeflaterOutputStream(buff, Deflater(), 0x10000), 0x40000))
+			val z = ZstdOutputStreamNoFinalizer(buff)
+			z.setLevel(6)
 
-			stream.writeInt(data.geometry.size.x)
+			try {
-			stream.writeInt(data.geometry.size.y)
+				val stream = DataOutputStream(BufferedOutputStream(z, 0x40000))
-			stream.writeBoolean(data.geometry.loopX)
-			stream.writeBoolean(data.geometry.loopY)
-			stream.writeJsonElement(data.data.content)
 
-			stream.close()
+				stream.writeInt(data.geometry.size.x)
-			writeMetadata("metadata", data.data.version ?: 0, buff.array.copyOf(buff.length))
+				stream.writeInt(data.geometry.size.y)
+				stream.writeBoolean(data.geometry.loopX)
+				stream.writeBoolean(data.geometry.loopY)
+				stream.writeJsonElement(data.data.content)
+
+				stream.close()
+				writeMetadata("metadata", data.data.version ?: 0, buff.array.copyOf(buff.length))
+			} finally {
+				z.close()
+			}
 		}
 	}