From c1d19d951d5ae6c6446ec09ae38289e624ffb426 Mon Sep 17 00:00:00 2001 From: DBotThePony Date: Thu, 28 Jul 2022 21:44:04 +0700 Subject: [PATCH] btreedb reading test --- .../kotlin/ru/dbotthepony/kstarbound/Main.kt | 208 ++++++------ .../ru/dbotthepony/kstarbound/Starbound.kt | 7 +- .../ru/dbotthepony/kstarbound/io/BTreeDB.kt | 300 ++++++++++++++++++ .../dbotthepony/kstarbound/io/BinaryJson.kt | 17 + .../ru/dbotthepony/kstarbound/io/Ext.kt | 57 ++++ 5 files changed, 471 insertions(+), 118 deletions(-) create mode 100644 src/main/kotlin/ru/dbotthepony/kstarbound/io/BTreeDB.kt diff --git a/src/main/kotlin/ru/dbotthepony/kstarbound/Main.kt b/src/main/kotlin/ru/dbotthepony/kstarbound/Main.kt index 964c4882..93497fbd 100644 --- a/src/main/kotlin/ru/dbotthepony/kstarbound/Main.kt +++ b/src/main/kotlin/ru/dbotthepony/kstarbound/Main.kt @@ -11,6 +11,7 @@ import ru.dbotthepony.kstarbound.client.StarboundClient import ru.dbotthepony.kstarbound.defs.TileDefinition import ru.dbotthepony.kstarbound.defs.projectile.ProjectilePhysics import ru.dbotthepony.kstarbound.defs.world.dungeon.DungeonWorldDef +import ru.dbotthepony.kstarbound.io.* import ru.dbotthepony.kstarbound.world.CHUNK_SIZE_FF import ru.dbotthepony.kstarbound.world.Chunk import ru.dbotthepony.kstarbound.world.ChunkPos @@ -18,8 +19,12 @@ import ru.dbotthepony.kstarbound.world.entities.Move import ru.dbotthepony.kstarbound.world.entities.PlayerEntity import ru.dbotthepony.kstarbound.world.entities.projectile.Projectile import ru.dbotthepony.kvector.vector.ndouble.Vector2d +import java.io.ByteArrayInputStream +import java.io.DataInputStream import java.io.File +import java.io.InputStream import java.util.* +import java.util.zip.Inflater private val LOGGER = LogManager.getLogger() @@ -34,6 +39,52 @@ fun main() { //return } + val db = BTreeDB(File("F:\\SteamLibrary\\steamapps\\common\\Starbound - Unstable\\storage\\universe\\389760395_938904237_-238610574_5.world")) + + /*if (true) { + val a = System.currentTimeMillis() + val worldMeta = db.read(byteArrayOf(0, 0, 0, 0, 0)) + println(System.currentTimeMillis() - a) + + val inflater = Inflater() + inflater.setInput(worldMeta!!) + + val output = ByteArray(1_000_000) + inflater.inflate(output) + + val stream = DataInputStream(ByteArrayInputStream(output)) + println("X tiles ${stream.readInt()}") + println("Y tiles ${stream.readInt()}") + + val metadata = VersionedJSON(stream) + println(metadata.data) + + return + }*/ + + /*if (true) { + val data = db.read(byteArrayOf(1, 0, 61, 0, 23)) + + val inflater = Inflater() + inflater.setInput(data!!) + + val output = ByteArray(64_000) + val actual = inflater.inflate(output) + File("F:\\SteamLibrary\\steamapps\\common\\Starbound - Unstable\\storage\\universe\\tiles.dat").writeBytes(output) + val reader = DataInputStream(ByteArrayInputStream(output)) + + reader.skipBytes(3) + + for (y in 0 .. 31) { + for (x in 0 .. 31) { + println("$x $y ${reader.readShort()}") + reader.skipBytes(29) + } + } + + return + }*/ + val client = StarboundClient() //Starbound.addFilePath(File("./unpacked_assets/")) @@ -47,138 +98,61 @@ fun main() { Starbound.terminateLoading = true } - var chunkA: Chunk<*, *>? = null - val ent = PlayerEntity(client.world!!) Starbound.onInitialize { - client.world!!.parallax = Starbound.parallaxAccess["barren"] - chunkA = client.world!!.computeIfAbsent(ChunkPos(0, 0)).chunk - val chunkB = client.world!!.computeIfAbsent(ChunkPos(-1, 0)).chunk - val chunkC = client.world!!.computeIfAbsent(ChunkPos(-2, 0)).chunk + for (chunkX in 0 .. 61) { + for (chunkY in 0 .. 61) { + val data = db.read(byteArrayOf(1, 0, chunkX.toByte(), 0, chunkY.toByte())) - val tile = Starbound.getTileDefinition("alienrock") + if (data != null) { + val chunk = client.world!!.computeIfAbsent(ChunkPos(chunkX, chunkY)) + val inflater = Inflater() + inflater.setInput(data) - for (x in -6 .. 6) { - for (y in 0 .. 4) { - val chnk = client.world!!.computeIfAbsent(ChunkPos(x, y)) + val output = ByteArray(64_000) + val actual = inflater.inflate(output) + val reader = DataInputStream(ByteArrayInputStream(output)) - if (y == 0) { - for (bx in 0 .. 31) { - for (by in 0 .. 3) { - chnk.chunk.foreground[bx, by] = tile + reader.skipBytes(3) + + var hitTile = false + + for (y in 0 .. 31) { + for (x in 0 .. 31) { + val materialID = reader.readShort() + val getMat = Starbound.tilesAccessID[materialID.toInt()] + + if (getMat != null) { + chunk.chunk.foreground[x, y] = getMat + hitTile = true + } + + reader.skipBytes(5) + + val materialID2 = reader.readShort() + val getMat2 = Starbound.tilesAccessID[materialID2.toInt()] + + if (getMat2 != null) { + chunk.chunk.background[x, y] = getMat2 + hitTile = true + } + + reader.skipBytes(22) } } - } - } - } - run { - val chunk = client.world!!.computeIfAbsent(ChunkPos(-2, 0)).chunk - - for (y in 0 .. CHUNK_SIZE_FF) { - for (x in 0 .. (CHUNK_SIZE_FF - y)) { - chunk.foreground[x, y] = tile - } - } - } - - run { - val chunk = client.world!!.computeIfAbsent(ChunkPos(-3, 0)).chunk - - for (y in 0 .. CHUNK_SIZE_FF) { - for (x in 0 .. (CHUNK_SIZE_FF - y * 2)) { - chunk.foreground[x, y] = tile - } - } - } - - for (x in 0 .. 31) { - for (y in 0 .. 3) { - chunkA!!.foreground[x, y] = tile - } - } - - for (x in 0 .. 31) { - for (y in 8 .. 9) { - chunkA!!.foreground[x, y] = tile - } - } - - for (x in 0 .. 31) { - for (y in 0 .. 0) { - chunkB.foreground[x, y] = tile - } - } - - for (x in 4 .. 8) { - for (y in 4 .. 8) { - chunkA!!.foreground[x, y] = null as TileDefinition? - } - } - - chunkA!!.foreground[18, 14] = tile - - /*val rand = Random() - - for (i in 0 .. 400) { - chunkA!!.foreground[rand.nextInt(0, CHUNK_SIZE_FF), rand.nextInt(0, CHUNK_SIZE_FF)] = tile - }*/ - - // ent.movement.dropToFloor() - - run { - var i = 0 - - for (proj in Starbound.projectilesAccess.values) { - if (proj.physics == ProjectilePhysics.BOUNCY) { - val projEnt = Projectile(client.world!!, proj) - projEnt.position = Vector2d(i * 2.0, 18.0) - projEnt.spawn() - i++ - } - } - } - - for (i in 0 .. 10) { - client.world!!.timer(i * 1.0, 1) { - val projEnt = Projectile(client.world!!, Starbound.projectilesAccess["pill"]!!) - projEnt.position = Vector2d(i * 2.0 - 15.0, 13.0) - projEnt.spawn() - } - } - - run { - val stripes = 0 - - for (stripe in 0 until stripes) { - for (x in 0 .. (stripes - stripe)) { - val movingBody = client.world!!.physics.createBody(BodyDef( - type = BodyType.DYNAMIC, - position = Vector2d(x = (-stripes + stripe) * 1.0 + x * 2.1, y = 8.0 + stripe * 2.1), - gravityScale = 1.1 - )) - - val dynamicBox: IShape<*> - - if (false) { - dynamicBox = PolygonShape() - dynamicBox.setAsBox(1.0, 1.0) - } else { - dynamicBox = CircleShape(1.0) + if (hitTile) { + //println(chunk.chunk.posVector2d) + // ent.position = chunk.chunk.posVector2d + Vector2d(16.0, 34.0) } - - movingBody.createFixture(FixtureDef( - shape = dynamicBox, - density = 1.0, - friction = 0.3 - )) } } } } - ent.position += Vector2d(y = 14.0, x = -10.0) + //ent.position += Vector2d(y = 14.0, x = -10.0) + ent.position = Vector2d(128.0 + 16.0, 672.0 + 48.0) client.onDrawGUI { client.gl.font.render("${ent.position}", y = 100f, scale = 0.25f) diff --git a/src/main/kotlin/ru/dbotthepony/kstarbound/Starbound.kt b/src/main/kotlin/ru/dbotthepony/kstarbound/Starbound.kt index 07fac8cc..b82916f5 100644 --- a/src/main/kotlin/ru/dbotthepony/kstarbound/Starbound.kt +++ b/src/main/kotlin/ru/dbotthepony/kstarbound/Starbound.kt @@ -1,6 +1,7 @@ package ru.dbotthepony.kstarbound import com.google.gson.* +import it.unimi.dsi.fastutil.ints.Int2ObjectAVLTreeMap import org.apache.logging.log4j.LogManager import ru.dbotthepony.kstarbound.api.IVFS import ru.dbotthepony.kstarbound.api.PhysicalFS @@ -38,11 +39,13 @@ object Starbound : IVFS { private val LOGGER = LogManager.getLogger() private val tiles = HashMap() + private val tilesByMaterialID = Int2ObjectAVLTreeMap() private val projectiles = HashMap() private val parallax = HashMap() private val functions = HashMap() val tilesAccess: Map = Collections.unmodifiableMap(tiles) + val tilesAccessID: Map = Collections.unmodifiableMap(tilesByMaterialID) val projectilesAccess: Map = Collections.unmodifiableMap(projectiles) val parallaxAccess: Map = Collections.unmodifiableMap(parallax) val functionsAccess: Map = Collections.unmodifiableMap(functions) @@ -230,7 +233,9 @@ object Starbound : IVFS { val tileDef = TileDefinitionBuilder.fromJson(JsonParser.parseReader(getReader(listedFile)) as JsonObject).build("/tiles/materials") - check(tiles[tileDef.materialName] == null) { "Already has material with ID ${tileDef.materialName} loaded!" } + check(tiles[tileDef.materialName] == null) { "Already has material with name ${tileDef.materialName} loaded!" } + check(tilesByMaterialID[tileDef.materialId] == null) { "Already has material with ID ${tileDef.materialId} loaded!" } + tilesByMaterialID[tileDef.materialId] = tileDef tiles[tileDef.materialName] = tileDef } catch (err: Throwable) { //throw TileDefLoadingException("Loading tile file $listedFile", err) diff --git a/src/main/kotlin/ru/dbotthepony/kstarbound/io/BTreeDB.kt b/src/main/kotlin/ru/dbotthepony/kstarbound/io/BTreeDB.kt new file mode 100644 index 00000000..88b9208b --- /dev/null +++ b/src/main/kotlin/ru/dbotthepony/kstarbound/io/BTreeDB.kt @@ -0,0 +1,300 @@ +package ru.dbotthepony.kstarbound.io + +import com.google.gson.JsonElement +import it.unimi.dsi.fastutil.ints.IntArraySet +import java.io.ByteArrayInputStream +import java.io.DataInputStream +import java.io.File +import java.io.InputStream +import java.io.RandomAccessFile + +private fun readHeader(reader: RandomAccessFile, required: Char) { + val read = reader.read() + require(read.toChar() == required) { "Bad Starbound Pak header, expected ${required.code}, got $read" } +} + +enum class TreeBlockType(val identity: String) { + INDEX("II"), + LEAF("LL"), + FREE("FF"); + + companion object { + operator fun get(index: String): TreeBlockType { + return when (index) { + INDEX.identity -> INDEX + LEAF.identity -> LEAF + FREE.identity -> FREE + else -> throw NoSuchElementException("Unknown block type $index") + } + } + } +} + +private operator fun ByteArray.compareTo(b: ByteArray): Int { + require(size == b.size) { "Keys are not of same size (${size} vs ${b.size})" } + + for (i in indices) { + if (this[i] > b[i]) { + return 1 + } else if (this[i] < b[i]) { + return -1 + } + } + + return 0 +} + +/** + * Класс, который позволяет читать и записывать в файлы Starbound BTReeDB 5 + * + * Big credit for https://github.com/blixt/py-starbound/blob/master/FORMATS.md#btreedb5 ! + */ +class BTreeDB(val path: File) { + val reader = RandomAccessFile(path, "r") + + init { + readHeader(reader, 'B') + readHeader(reader, 'T') + readHeader(reader, 'r') + readHeader(reader, 'e') + readHeader(reader, 'e') + readHeader(reader, 'D') + readHeader(reader, 'B') + readHeader(reader, '5') + } + + val blockSize = reader.readInt().toLong() + val dbNameRaw = ByteArray(16).also { reader.read(it) } + val indexKeySize = reader.readInt() + val useNodeTwo = reader.readBoolean() + val freeNodeIndex1 = reader.readInt().toLong() + + init { reader.skipBytes(4) } + val freeBlockOffset1 = reader.readInt().toLong() + val rootNode1Index = reader.readInt().toLong() + val rootNode1IsLeaf = reader.readBoolean() + val freeNodeIndex2 = reader.readInt().toLong() + init { reader.skipBytes(4) } + val freeBlockOffset2 = reader.readInt().toLong() + val rootNode2Index = reader.readInt().toLong() + val rootNode2IsLeaf = reader.readBoolean() + init { reader.skipBytes(445) } + + val blocksOffsetStart = reader.filePointer + + init { + // check(reader.length() - blocksOffsetStart == 512L) { "Unexpected header size of ${reader.length() - blocksOffsetStart} bytes" } + check((reader.length() - 512L) % blockSize == 0L) { "Junk data somewhere in file (${(reader.length() - 512L) % blockSize} lingering bytes)" } + } + + val rootNodeIndex get() = if (useNodeTwo) rootNode2Index else rootNode1Index + val rootNodeIsLeaf get() = if (useNodeTwo) rootNode2IsLeaf else rootNode1IsLeaf + + fun readBlockType() = TreeBlockType[reader.readASCIIString(2)] + + fun findAllKeys(index: Long = rootNodeIndex): List { + seekBlock(index) + + val list = ArrayList() + val type = readBlockType() + + if (type == TreeBlockType.LEAF) { + val keyAmount = reader.readInt() + // offset внутри лепестка в байтах + var offset = 6 + + for (i in 0 until keyAmount) { + // читаем ключ + list.add(ByteArray(indexKeySize).also { reader.read(it) }) + offset += indexKeySize + + // читаем размер данных внутри ключа + var (dataLength, readBytes) = reader.readVarIntInfo() + offset += readBytes + + while (true) { + // если конец данных внутри текущего блока, останавливаемся + if (offset + dataLength <= blockSize - 4) { + reader.skipBytes(dataLength) + offset += dataLength + break + } + + // иначе, ищем следующий блок + + // пропускаем оставшиеся данные, переходим на границу текущего блока-лепестка + val delta = (blockSize - 4 - offset).toInt() + reader.skipBytes(delta) + + // ищем следующий блок с нашими данными + val nextBlockIndex = reader.readInt() + seekBlock(nextBlockIndex.toLong()) + + // удостоверяемся что мы попали в лепесток + check(readBlockType() == TreeBlockType.LEAF) { "Did not hit leaf block" } + offset = 2 + dataLength -= delta + } + } + } else if (type == TreeBlockType.INDEX) { + reader.skipBytes(1) + val keyAmount = reader.readInt() + + val blockList = IntArraySet() + blockList.add(reader.readInt()) + + for (i in 0 until keyAmount) { + // ключ + reader.skipBytes(indexKeySize) + + // указатель на блок + blockList.add(reader.readInt()) + } + + // читаем все дочерние блоки на ключи + for (block in blockList.intIterator()) { + for (key in findAllKeys(block.toLong())) { + list.add(key) + } + } + } + + return list + } + + fun read(key: ByteArray): ByteArray? { + require(key.size == indexKeySize) { "Key provided is ${key.size} in size, while $indexKeySize is required" } + + seekBlock(rootNodeIndex) + var type = readBlockType() + var iterations = 1000 + + val keyLoader = ByteArray(indexKeySize) + + // сканирование индекса + while (iterations-- > 0 && type != TreeBlockType.LEAF) { + if (type == TreeBlockType.FREE) { + throw IllegalStateException("Hit free block while scanning index for ${key.joinToString(", ")}") + } + + reader.skipBytes(1) + + val keyCount = reader.readInt() + // if keyAmount == 4 then + // B a B b B c B d B + val readKeys = ByteArray((keyCount + 1) * 4 + keyCount * indexKeySize) + reader.readFully(readKeys) + + val stream = DataInputStream(ByteArrayInputStream(readKeys)) + + var read = false + + // B a + // B b + // B c + // B d + for (keyIndex in 0 until keyCount) { + // указатель на левый блок + val pointer = stream.readInt() + + // левый ключ, всё что меньше него находится в левом блоке + stream.readFully(keyLoader) + + // нужный ключ меньше самого первого ключа, поэтому он находится где то в левом блоке + if (key < keyLoader) { + seekBlock(pointer.toLong()) + type = readBlockType() + read = true + break + } + } + + if (!read) { + // ... B + seekBlock(stream.readInt().toLong()) + type = readBlockType() + } + } + + // мы пришли в лепесток, теперь прямолинейно ищем в linked list + var offset = 6 + val keyCount = reader.readInt() + + for (keyIndex in 0 until keyCount) { + // читаем ключ + reader.read(keyLoader) + offset += indexKeySize + + // читаем размер данных + var (dataLength, readBytes) = reader.readVarIntInfo() + offset += readBytes + + // это наш блок + if (keyLoader.contentEquals(key)) { + val binary = ByteArray(dataLength) + var binaryOffset = 0 + + // читаем данные + while (true) { + // если конец данных внутри текущего блока, останавливаемся + if (offset + dataLength <= blockSize - 4) { + reader.readFully(binary, binaryOffset, dataLength) + offset += dataLength + binaryOffset += dataLength + break + } + + // иначе, ищем следующий блок + + // пропускаем оставшиеся данные, переходим на границу текущего блока-лепестка + val delta = (blockSize - 4 - offset).toInt() + reader.readFully(binary, binaryOffset, delta) + binaryOffset += delta + + // ищем следующий блок с нашими данными + val nextBlockIndex = reader.readInt() + seekBlock(nextBlockIndex.toLong()) + + // удостоверяемся что мы попали в лепесток + check(readBlockType() == TreeBlockType.LEAF) { "Did not hit leaf block" } + offset = 2 + dataLength -= delta + } + + return binary + } else { + // это не наш блок, пропускаем его + while (true) { + // если конец данных внутри текущего блока, останавливаемся + if (offset + dataLength <= blockSize - 4) { + reader.skipBytes(dataLength) + offset += dataLength + break + } + + // иначе, ищем следующий блок + + // пропускаем оставшиеся данные, переходим на границу текущего блока-лепестка + val delta = (blockSize - 4 - offset).toInt() + reader.skipBytes(delta) + + // ищем следующий блок с нашими данными + val nextBlockIndex = reader.readInt() + seekBlock(nextBlockIndex.toLong()) + + // удостоверяемся что мы попали в лепесток + check(readBlockType() == TreeBlockType.LEAF) { "Did not hit leaf block" } + offset = 2 + dataLength -= delta + } + } + } + + return null + } + + fun seekBlock(id: Long) { + require(id >= 0) { "Negative id $id" } + reader.seek(id * blockSize + blocksOffsetStart) + } +} diff --git a/src/main/kotlin/ru/dbotthepony/kstarbound/io/BinaryJson.kt b/src/main/kotlin/ru/dbotthepony/kstarbound/io/BinaryJson.kt index d833aa59..afe7197f 100644 --- a/src/main/kotlin/ru/dbotthepony/kstarbound/io/BinaryJson.kt +++ b/src/main/kotlin/ru/dbotthepony/kstarbound/io/BinaryJson.kt @@ -172,3 +172,20 @@ object BinaryJson { return build } } + +class VersionedJSON(var name: String = "Versioned JSON") { + var isVersioned = false + var version = 0 + var data: JsonElement? = null + + constructor(stream: DataInputStream) : this() { + name = stream.readASCIIString(stream.readVarInt()) + isVersioned = stream.readBoolean() + + if (isVersioned) { + version = stream.readInt() + } + + data = BinaryJson.readElement(stream) + } +} diff --git a/src/main/kotlin/ru/dbotthepony/kstarbound/io/Ext.kt b/src/main/kotlin/ru/dbotthepony/kstarbound/io/Ext.kt index f44e1f83..dba92a09 100644 --- a/src/main/kotlin/ru/dbotthepony/kstarbound/io/Ext.kt +++ b/src/main/kotlin/ru/dbotthepony/kstarbound/io/Ext.kt @@ -1,5 +1,6 @@ package ru.dbotthepony.kstarbound.io +import it.unimi.dsi.fastutil.bytes.ByteArrayList import java.io.DataInputStream import java.io.IOException import java.io.InputStream @@ -25,6 +26,8 @@ fun RandomAccessFile.readVarLong(): Long { return result } +data class VarIntReadResult(val value: Int, val cells: Int) + /** * Читает Variable Length Integer как Int */ @@ -45,6 +48,28 @@ fun RandomAccessFile.readVarInt(): Int { return result } +/** + * Читает Variable Length Integer как Int + */ +fun RandomAccessFile.readVarIntInfo(): VarIntReadResult { + var result = 0 + var read = read() + var i = 1 + + while (true) { + result = (result shl 7) or (read and 0x7F) + + if (read and 0x80 == 0) { + break + } + + read = read() + i++ + } + + return VarIntReadResult(result, i) +} + /** * Читает Variable Length Integer как Long */ @@ -112,3 +137,35 @@ fun InputStream.readASCIIString(length: Int): String { return bytes.toString(Charsets.UTF_8) } + +fun RandomAccessFile.readCString(): String { + val bytes = ByteArrayList() + var read = read() + + while (read != 0) { + bytes.add(read.toByte()) + read = read() + } + + return ByteArray(bytes.size).also { + for (i in it.indices) { + it[i] = bytes.getByte(i) + } + }.toString(Charsets.UTF_8) +} + +fun InputStream.readCString(): String { + val bytes = ByteArrayList() + var read = read() + + while (read != 0) { + bytes.add(read.toByte()) + read = read() + } + + return ByteArray(bytes.size).also { + for (i in it.indices) { + it[i] = bytes.getByte(i) + } + }.toString(Charsets.UTF_8) +}