DBot/KStarbound
DBot/KStarbound
3
0
Fork 0
Code Issues 1 Pull Requests Packages Projects Releases Wiki Activity

Move universe storage to zstd

Browse Source
This commit is contained in:
DBotThePony 2024-12-09 14:15:49 +07:00
parent c5fa4e4b59
commit 11e4efb2af
Signed by: DBot
GPG Key ID: DCC23B5715498507
3 changed files with 22 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
src/main/kotlin/ru/dbotthepony/kstarbound/json/BinaryJsonWriter.kt
View File

@ -21,7 +21,7 @@ private enum class DeflateType {
NONE NONE
} }
private fun <T> T.callWrite(deflate: DeflateType, callable: DataOutputStream.(T) -> Unit): ByteArray { private fun <T> T.callWrite(deflate: DeflateType, zstdCompressionLevel: Int = 6, callable: DataOutputStream.(T) -> Unit): ByteArray {
val stream = FastByteArrayOutputStream() val stream = FastByteArrayOutputStream()
when (deflate) { when (deflate) {
@ -36,7 +36,7 @@ private fun <T> T.callWrite(deflate: DeflateType, callable: DataOutputStream.(T)
DeflateType.ZSTD -> { DeflateType.ZSTD -> {
val s = ZstdOutputStreamNoFinalizer(stream) val s = ZstdOutputStreamNoFinalizer(stream)
s.setLevel(6) s.setLevel(zstdCompressionLevel)
DataOutputStream(BufferedOutputStream(s, 0x10000)).use { DataOutputStream(BufferedOutputStream(s, 0x10000)).use {
callable(it, this) callable(it, this)
@ -57,9 +57,9 @@ fun JsonElement.writeJsonElementDeflated(): ByteArray = callWrite(DeflateType.ZL
fun JsonObject.writeJsonObjectDeflated(): ByteArray = callWrite(DeflateType.ZLIB) { writeJsonObject(it) } fun JsonObject.writeJsonObjectDeflated(): ByteArray = callWrite(DeflateType.ZLIB) { writeJsonObject(it) }
fun JsonArray.writeJsonArrayDeflated(): ByteArray = callWrite(DeflateType.ZLIB) { writeJsonArray(it) } fun JsonArray.writeJsonArrayDeflated(): ByteArray = callWrite(DeflateType.ZLIB) { writeJsonArray(it) }
fun JsonElement.writeJsonElementZstd(): ByteArray = callWrite(DeflateType.ZSTD) { writeJsonElement(it) } fun JsonElement.writeJsonElementZstd(level: Int = 6): ByteArray = callWrite(DeflateType.ZSTD, zstdCompressionLevel = level) { writeJsonElement(it) }
fun JsonObject.writeJsonObjectZstd(): ByteArray = callWrite(DeflateType.ZSTD) { writeJsonObject(it) } fun JsonObject.writeJsonObjectZstd(level: Int = 6): ByteArray = callWrite(DeflateType.ZSTD, zstdCompressionLevel = level) { writeJsonObject(it) }
fun JsonArray.writeJsonArrayZstd(): ByteArray = callWrite(DeflateType.ZSTD) { writeJsonArray(it) } fun JsonArray.writeJsonArrayZstd(level: Int = 6): ByteArray = callWrite(DeflateType.ZSTD, zstdCompressionLevel = level) { writeJsonArray(it) }
fun DataOutputStream.writeJsonElement(value: JsonElement) { fun DataOutputStream.writeJsonElement(value: JsonElement) {
when (value) { when (value) {

6
src/main/kotlin/ru/dbotthepony/kstarbound/server/ServerConnection.kt
View File

@ -742,13 +742,13 @@ class ServerConnection(val server: StarboundServer, type: ConnectionType) : Conn
val startingLocation = findStartingSystem() ?: return val startingLocation = findStartingSystem() ?: return
scope.launch { shipFlightEventLoop(startingLocation.location, SystemWorldLocation.Celestial(startingLocation)) } scope.launch { shipFlightEventLoop(startingLocation.location, SystemWorldLocation.Celestial(startingLocation)) }
} else { } else {
/*if (context.returnWarp != null) { if (context.returnWarp != null) {
enqueueWarp(context.returnWarp, ifFailed = WarpAlias.OwnShip) enqueueWarp(context.returnWarp, ifFailed = WarpAlias.OwnShip)
} else { } else {
enqueueWarp(WarpAlias.OwnShip) enqueueWarp(WarpAlias.OwnShip)
}*/ }
enqueueWarp(WarpAction.World(WorldID.Instance("outpost"))) //enqueueWarp(WarpAction.World(WorldID.Instance("outpost")))
scope.launch { shipFlightEventLoop(context.shipCoordinate, context.systemLocation) } scope.launch { shipFlightEventLoop(context.shipCoordinate, context.systemLocation) }
} }

32
src/main/kotlin/ru/dbotthepony/kstarbound/server/world/ServerUniverse.kt
View File

@ -8,19 +8,15 @@ import it.unimi.dsi.fastutil.objects.ObjectArrayList
import it.unimi.dsi.fastutil.objects.ObjectOpenHashSet import it.unimi.dsi.fastutil.objects.ObjectOpenHashSet
import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.SupervisorJob import kotlinx.coroutines.SupervisorJob
import kotlinx.coroutines.asCoroutineDispatcher
import kotlinx.coroutines.async import kotlinx.coroutines.async
import kotlinx.coroutines.cancel import kotlinx.coroutines.cancel
import kotlinx.coroutines.delay
import kotlinx.coroutines.future.asCompletableFuture import kotlinx.coroutines.future.asCompletableFuture
import kotlinx.coroutines.future.await import kotlinx.coroutines.future.await
import ru.dbotthepony.kommons.gson.JsonArrayCollector import ru.dbotthepony.kommons.gson.JsonArrayCollector
import ru.dbotthepony.kommons.gson.contains import ru.dbotthepony.kommons.gson.contains
import ru.dbotthepony.kommons.gson.get import ru.dbotthepony.kommons.gson.get
import ru.dbotthepony.kommons.gson.set import ru.dbotthepony.kommons.gson.set
import ru.dbotthepony.kstarbound.math.AABBi
import ru.dbotthepony.kommons.util.KOptional import ru.dbotthepony.kommons.util.KOptional
import ru.dbotthepony.kstarbound.math.vector.Vector2i
import ru.dbotthepony.kstarbound.Globals import ru.dbotthepony.kstarbound.Globals
import ru.dbotthepony.kstarbound.Starbound import ru.dbotthepony.kstarbound.Starbound
import ru.dbotthepony.kstarbound.defs.world.CelestialBaseInformation import ru.dbotthepony.kstarbound.defs.world.CelestialBaseInformation
@ -30,11 +26,13 @@ import ru.dbotthepony.kstarbound.fromJson
import ru.dbotthepony.kstarbound.io.BTreeDB5 import ru.dbotthepony.kstarbound.io.BTreeDB5
import ru.dbotthepony.kstarbound.json.jsonArrayOf import ru.dbotthepony.kstarbound.json.jsonArrayOf
import ru.dbotthepony.kstarbound.json.mergeJson import ru.dbotthepony.kstarbound.json.mergeJson
import ru.dbotthepony.kstarbound.json.readJsonArrayInflated import ru.dbotthepony.kstarbound.json.readJsonArrayZstd
import ru.dbotthepony.kstarbound.json.readJsonElementInflated import ru.dbotthepony.kstarbound.json.readJsonElementZstd
import ru.dbotthepony.kstarbound.json.writeJsonArrayDeflated import ru.dbotthepony.kstarbound.json.writeJsonArrayZstd
import ru.dbotthepony.kstarbound.json.writeJsonElementDeflated import ru.dbotthepony.kstarbound.json.writeJsonElementZstd
import ru.dbotthepony.kstarbound.math.AABBi
import ru.dbotthepony.kstarbound.math.Line2d import ru.dbotthepony.kstarbound.math.Line2d
import ru.dbotthepony.kstarbound.math.vector.Vector2i
import ru.dbotthepony.kstarbound.math.vector.Vector3i import ru.dbotthepony.kstarbound.math.vector.Vector3i
import ru.dbotthepony.kstarbound.util.CarriedExecutor import ru.dbotthepony.kstarbound.util.CarriedExecutor
import ru.dbotthepony.kstarbound.util.ScheduledCoroutineExecutor import ru.dbotthepony.kstarbound.util.ScheduledCoroutineExecutor
@ -57,11 +55,9 @@ import java.sql.ResultSet
import java.util.concurrent.CompletableFuture import java.util.concurrent.CompletableFuture
import java.util.concurrent.ConcurrentHashMap import java.util.concurrent.ConcurrentHashMap
import java.util.concurrent.TimeUnit import java.util.concurrent.TimeUnit
import java.util.function.Consumer
import java.util.function.Function import java.util.function.Function
import java.util.function.Supplier import java.util.function.Supplier
import java.util.random.RandomGenerator import java.util.random.RandomGenerator
import kotlin.collections.ArrayList
class ServerUniverse(folder: File? = null) : Universe(), Closeable { class ServerUniverse(folder: File? = null) : Universe(), Closeable {
override val baseInformation: CelestialBaseInformation override val baseInformation: CelestialBaseInformation
@ -148,8 +144,8 @@ class ServerUniverse(folder: File? = null) : Universe(), Closeable {
private data class Chunk(val x: Int, val y: Int, val systems: Set<Vector3i>, val constellations: Set<Pair<Vector2i, Vector2i>>) { private data class Chunk(val x: Int, val y: Int, val systems: Set<Vector3i>, val constellations: Set<Pair<Vector2i, Vector2i>>) {
constructor(x: Int, y: Int, data: ResultSet) : this( constructor(x: Int, y: Int, data: ResultSet) : this(
x, y, x, y,
data.getBytes(1).readJsonArrayInflated().map { Vector3i(it.asJsonArray[0].asInt, it.asJsonArray[1].asInt, it.asJsonArray[2].asInt) }.toSet(), data.getBytes(1).readJsonArrayZstd().map { Vector3i(it.asJsonArray[0].asInt, it.asJsonArray[1].asInt, it.asJsonArray[2].asInt) }.toSet(),
data.getBytes(2).readJsonArrayInflated().map { data.getBytes(2).readJsonArrayZstd().map {
val a = it.asJsonArray[0].asJsonArray val a = it.asJsonArray[0].asJsonArray
val b = it.asJsonArray[1].asJsonArray val b = it.asJsonArray[1].asJsonArray
Vector2i(a[0].asInt, a[1].asInt) to Vector2i(b[0].asInt, b[1].asInt) Vector2i(a[0].asInt, a[1].asInt) to Vector2i(b[0].asInt, b[1].asInt)
@ -162,10 +158,10 @@ class ServerUniverse(folder: File? = null) : Universe(), Closeable {
systems.stream() systems.stream()
.map { jsonArrayOf(it.x, it.y, it.z) } .map { jsonArrayOf(it.x, it.y, it.z) }
.collect(JsonArrayCollector) .collect(JsonArrayCollector)
.writeJsonArrayDeflated(), .writeJsonArrayZstd(4),
constellations.stream().map { constellations.stream().map {
jsonArrayOf(jsonArrayOf(it.first.x, it.first.y), jsonArrayOf(it.second.x, it.second.y)) jsonArrayOf(jsonArrayOf(it.first.x, it.first.y), jsonArrayOf(it.second.x, it.second.y))
}.collect(JsonArrayCollector).writeJsonArrayDeflated() }.collect(JsonArrayCollector).writeJsonArrayZstd(4)
) )
} }
@ -198,10 +194,10 @@ class ServerUniverse(folder: File? = null) : Universe(), Closeable {
fun serialize(): SerializedSystem { fun serialize(): SerializedSystem {
return SerializedSystem( return SerializedSystem(
x, y, z, x, y, z,
Starbound.gson.toJsonTree(parameters).writeJsonElementDeflated(), Starbound.gson.toJsonTree(parameters).writeJsonElementZstd(8),
planets.entries.stream() planets.entries.stream()
.map { jsonArrayOf(it.key.first, it.key.second, it.value) } .map { jsonArrayOf(it.key.first, it.key.second, it.value) }
.collect(JsonArrayCollector).writeJsonArrayDeflated() .collect(JsonArrayCollector).writeJsonArrayZstd(8)
) )
} }
@ -270,9 +266,9 @@ class ServerUniverse(folder: File? = null) : Universe(), Closeable {
// deserialize in off-thread since it involves big json structures // deserialize in off-thread since it involves big json structures
Starbound.EXECUTOR.supplyAsync { Starbound.EXECUTOR.supplyAsync {
val parameters: CelestialParameters = Starbound.gson.fromJson(parametersBytes.readJsonElementInflated())!! val parameters: CelestialParameters = Starbound.gson.fromJson(parametersBytes.readJsonElementZstd())!!
val planets: Map<Pair<Int, Int>, CelestialParameters> = planetsBytes.readJsonArrayInflated().associate { val planets: Map<Pair<Int, Int>, CelestialParameters> = planetsBytes.readJsonArrayZstd().associate {
it as JsonArray it as JsonArray
(it[0].asInt to it[1].asInt) to Starbound.gson.fromJson(it[2])!! (it[0].asInt to it[1].asInt) to Starbound.gson.fromJson(it[2])!!
} }