btreedb reading test

This commit is contained in:
DBotThePony 2022-07-28 21:44:04 +07:00
parent 067da35ada
commit c1d19d951d
Signed by: DBot
GPG Key ID: DCC23B5715498507
5 changed files with 471 additions and 118 deletions

View File

@ -11,6 +11,7 @@ import ru.dbotthepony.kstarbound.client.StarboundClient
import ru.dbotthepony.kstarbound.defs.TileDefinition
import ru.dbotthepony.kstarbound.defs.projectile.ProjectilePhysics
import ru.dbotthepony.kstarbound.defs.world.dungeon.DungeonWorldDef
import ru.dbotthepony.kstarbound.io.*
import ru.dbotthepony.kstarbound.world.CHUNK_SIZE_FF
import ru.dbotthepony.kstarbound.world.Chunk
import ru.dbotthepony.kstarbound.world.ChunkPos
@ -18,8 +19,12 @@ import ru.dbotthepony.kstarbound.world.entities.Move
import ru.dbotthepony.kstarbound.world.entities.PlayerEntity
import ru.dbotthepony.kstarbound.world.entities.projectile.Projectile
import ru.dbotthepony.kvector.vector.ndouble.Vector2d
import java.io.ByteArrayInputStream
import java.io.DataInputStream
import java.io.File
import java.io.InputStream
import java.util.*
import java.util.zip.Inflater
private val LOGGER = LogManager.getLogger()
@ -34,6 +39,52 @@ fun main() {
//return
}
val db = BTreeDB(File("F:\\SteamLibrary\\steamapps\\common\\Starbound - Unstable\\storage\\universe\\389760395_938904237_-238610574_5.world"))
/*if (true) {
val a = System.currentTimeMillis()
val worldMeta = db.read(byteArrayOf(0, 0, 0, 0, 0))
println(System.currentTimeMillis() - a)
val inflater = Inflater()
inflater.setInput(worldMeta!!)
val output = ByteArray(1_000_000)
inflater.inflate(output)
val stream = DataInputStream(ByteArrayInputStream(output))
println("X tiles ${stream.readInt()}")
println("Y tiles ${stream.readInt()}")
val metadata = VersionedJSON(stream)
println(metadata.data)
return
}*/
/*if (true) {
val data = db.read(byteArrayOf(1, 0, 61, 0, 23))
val inflater = Inflater()
inflater.setInput(data!!)
val output = ByteArray(64_000)
val actual = inflater.inflate(output)
File("F:\\SteamLibrary\\steamapps\\common\\Starbound - Unstable\\storage\\universe\\tiles.dat").writeBytes(output)
val reader = DataInputStream(ByteArrayInputStream(output))
reader.skipBytes(3)
for (y in 0 .. 31) {
for (x in 0 .. 31) {
println("$x $y ${reader.readShort()}")
reader.skipBytes(29)
}
}
return
}*/
val client = StarboundClient()
//Starbound.addFilePath(File("./unpacked_assets/"))
@ -47,138 +98,61 @@ fun main() {
Starbound.terminateLoading = true
}
var chunkA: Chunk<*, *>? = null
val ent = PlayerEntity(client.world!!)
Starbound.onInitialize {
client.world!!.parallax = Starbound.parallaxAccess["barren"]
chunkA = client.world!!.computeIfAbsent(ChunkPos(0, 0)).chunk
val chunkB = client.world!!.computeIfAbsent(ChunkPos(-1, 0)).chunk
val chunkC = client.world!!.computeIfAbsent(ChunkPos(-2, 0)).chunk
for (chunkX in 0 .. 61) {
for (chunkY in 0 .. 61) {
val data = db.read(byteArrayOf(1, 0, chunkX.toByte(), 0, chunkY.toByte()))
val tile = Starbound.getTileDefinition("alienrock")
if (data != null) {
val chunk = client.world!!.computeIfAbsent(ChunkPos(chunkX, chunkY))
val inflater = Inflater()
inflater.setInput(data)
for (x in -6 .. 6) {
for (y in 0 .. 4) {
val chnk = client.world!!.computeIfAbsent(ChunkPos(x, y))
val output = ByteArray(64_000)
val actual = inflater.inflate(output)
val reader = DataInputStream(ByteArrayInputStream(output))
if (y == 0) {
for (bx in 0 .. 31) {
for (by in 0 .. 3) {
chnk.chunk.foreground[bx, by] = tile
reader.skipBytes(3)
var hitTile = false
for (y in 0 .. 31) {
for (x in 0 .. 31) {
val materialID = reader.readShort()
val getMat = Starbound.tilesAccessID[materialID.toInt()]
if (getMat != null) {
chunk.chunk.foreground[x, y] = getMat
hitTile = true
}
reader.skipBytes(5)
val materialID2 = reader.readShort()
val getMat2 = Starbound.tilesAccessID[materialID2.toInt()]
if (getMat2 != null) {
chunk.chunk.background[x, y] = getMat2
hitTile = true
}
reader.skipBytes(22)
}
}
}
}
}
run {
val chunk = client.world!!.computeIfAbsent(ChunkPos(-2, 0)).chunk
for (y in 0 .. CHUNK_SIZE_FF) {
for (x in 0 .. (CHUNK_SIZE_FF - y)) {
chunk.foreground[x, y] = tile
}
}
}
run {
val chunk = client.world!!.computeIfAbsent(ChunkPos(-3, 0)).chunk
for (y in 0 .. CHUNK_SIZE_FF) {
for (x in 0 .. (CHUNK_SIZE_FF - y * 2)) {
chunk.foreground[x, y] = tile
}
}
}
for (x in 0 .. 31) {
for (y in 0 .. 3) {
chunkA!!.foreground[x, y] = tile
}
}
for (x in 0 .. 31) {
for (y in 8 .. 9) {
chunkA!!.foreground[x, y] = tile
}
}
for (x in 0 .. 31) {
for (y in 0 .. 0) {
chunkB.foreground[x, y] = tile
}
}
for (x in 4 .. 8) {
for (y in 4 .. 8) {
chunkA!!.foreground[x, y] = null as TileDefinition?
}
}
chunkA!!.foreground[18, 14] = tile
/*val rand = Random()
for (i in 0 .. 400) {
chunkA!!.foreground[rand.nextInt(0, CHUNK_SIZE_FF), rand.nextInt(0, CHUNK_SIZE_FF)] = tile
}*/
// ent.movement.dropToFloor()
run {
var i = 0
for (proj in Starbound.projectilesAccess.values) {
if (proj.physics == ProjectilePhysics.BOUNCY) {
val projEnt = Projectile(client.world!!, proj)
projEnt.position = Vector2d(i * 2.0, 18.0)
projEnt.spawn()
i++
}
}
}
for (i in 0 .. 10) {
client.world!!.timer(i * 1.0, 1) {
val projEnt = Projectile(client.world!!, Starbound.projectilesAccess["pill"]!!)
projEnt.position = Vector2d(i * 2.0 - 15.0, 13.0)
projEnt.spawn()
}
}
run {
val stripes = 0
for (stripe in 0 until stripes) {
for (x in 0 .. (stripes - stripe)) {
val movingBody = client.world!!.physics.createBody(BodyDef(
type = BodyType.DYNAMIC,
position = Vector2d(x = (-stripes + stripe) * 1.0 + x * 2.1, y = 8.0 + stripe * 2.1),
gravityScale = 1.1
))
val dynamicBox: IShape<*>
if (false) {
dynamicBox = PolygonShape()
dynamicBox.setAsBox(1.0, 1.0)
} else {
dynamicBox = CircleShape(1.0)
if (hitTile) {
//println(chunk.chunk.posVector2d)
// ent.position = chunk.chunk.posVector2d + Vector2d(16.0, 34.0)
}
movingBody.createFixture(FixtureDef(
shape = dynamicBox,
density = 1.0,
friction = 0.3
))
}
}
}
}
ent.position += Vector2d(y = 14.0, x = -10.0)
//ent.position += Vector2d(y = 14.0, x = -10.0)
ent.position = Vector2d(128.0 + 16.0, 672.0 + 48.0)
client.onDrawGUI {
client.gl.font.render("${ent.position}", y = 100f, scale = 0.25f)

View File

@ -1,6 +1,7 @@
package ru.dbotthepony.kstarbound
import com.google.gson.*
import it.unimi.dsi.fastutil.ints.Int2ObjectAVLTreeMap
import org.apache.logging.log4j.LogManager
import ru.dbotthepony.kstarbound.api.IVFS
import ru.dbotthepony.kstarbound.api.PhysicalFS
@ -38,11 +39,13 @@ object Starbound : IVFS {
private val LOGGER = LogManager.getLogger()
private val tiles = HashMap<String, TileDefinition>()
private val tilesByMaterialID = Int2ObjectAVLTreeMap<TileDefinition>()
private val projectiles = HashMap<String, ConfiguredProjectile>()
private val parallax = HashMap<String, ParallaxPrototype>()
private val functions = HashMap<String, JsonFunction>()
val tilesAccess: Map<String, TileDefinition> = Collections.unmodifiableMap(tiles)
val tilesAccessID: Map<Int, TileDefinition> = Collections.unmodifiableMap(tilesByMaterialID)
val projectilesAccess: Map<String, ConfiguredProjectile> = Collections.unmodifiableMap(projectiles)
val parallaxAccess: Map<String, ParallaxPrototype> = Collections.unmodifiableMap(parallax)
val functionsAccess: Map<String, JsonFunction> = Collections.unmodifiableMap(functions)
@ -230,7 +233,9 @@ object Starbound : IVFS {
val tileDef = TileDefinitionBuilder.fromJson(JsonParser.parseReader(getReader(listedFile)) as JsonObject).build("/tiles/materials")
check(tiles[tileDef.materialName] == null) { "Already has material with ID ${tileDef.materialName} loaded!" }
check(tiles[tileDef.materialName] == null) { "Already has material with name ${tileDef.materialName} loaded!" }
check(tilesByMaterialID[tileDef.materialId] == null) { "Already has material with ID ${tileDef.materialId} loaded!" }
tilesByMaterialID[tileDef.materialId] = tileDef
tiles[tileDef.materialName] = tileDef
} catch (err: Throwable) {
//throw TileDefLoadingException("Loading tile file $listedFile", err)

View File

@ -0,0 +1,300 @@
package ru.dbotthepony.kstarbound.io
import com.google.gson.JsonElement
import it.unimi.dsi.fastutil.ints.IntArraySet
import java.io.ByteArrayInputStream
import java.io.DataInputStream
import java.io.File
import java.io.InputStream
import java.io.RandomAccessFile
private fun readHeader(reader: RandomAccessFile, required: Char) {
val read = reader.read()
require(read.toChar() == required) { "Bad Starbound Pak header, expected ${required.code}, got $read" }
}
enum class TreeBlockType(val identity: String) {
INDEX("II"),
LEAF("LL"),
FREE("FF");
companion object {
operator fun get(index: String): TreeBlockType {
return when (index) {
INDEX.identity -> INDEX
LEAF.identity -> LEAF
FREE.identity -> FREE
else -> throw NoSuchElementException("Unknown block type $index")
}
}
}
}
private operator fun ByteArray.compareTo(b: ByteArray): Int {
require(size == b.size) { "Keys are not of same size (${size} vs ${b.size})" }
for (i in indices) {
if (this[i] > b[i]) {
return 1
} else if (this[i] < b[i]) {
return -1
}
}
return 0
}
/**
* Класс, который позволяет читать и записывать в файлы Starbound BTReeDB 5
*
* Big credit for https://github.com/blixt/py-starbound/blob/master/FORMATS.md#btreedb5 !
*/
class BTreeDB(val path: File) {
val reader = RandomAccessFile(path, "r")
init {
readHeader(reader, 'B')
readHeader(reader, 'T')
readHeader(reader, 'r')
readHeader(reader, 'e')
readHeader(reader, 'e')
readHeader(reader, 'D')
readHeader(reader, 'B')
readHeader(reader, '5')
}
val blockSize = reader.readInt().toLong()
val dbNameRaw = ByteArray(16).also { reader.read(it) }
val indexKeySize = reader.readInt()
val useNodeTwo = reader.readBoolean()
val freeNodeIndex1 = reader.readInt().toLong()
init { reader.skipBytes(4) }
val freeBlockOffset1 = reader.readInt().toLong()
val rootNode1Index = reader.readInt().toLong()
val rootNode1IsLeaf = reader.readBoolean()
val freeNodeIndex2 = reader.readInt().toLong()
init { reader.skipBytes(4) }
val freeBlockOffset2 = reader.readInt().toLong()
val rootNode2Index = reader.readInt().toLong()
val rootNode2IsLeaf = reader.readBoolean()
init { reader.skipBytes(445) }
val blocksOffsetStart = reader.filePointer
init {
// check(reader.length() - blocksOffsetStart == 512L) { "Unexpected header size of ${reader.length() - blocksOffsetStart} bytes" }
check((reader.length() - 512L) % blockSize == 0L) { "Junk data somewhere in file (${(reader.length() - 512L) % blockSize} lingering bytes)" }
}
val rootNodeIndex get() = if (useNodeTwo) rootNode2Index else rootNode1Index
val rootNodeIsLeaf get() = if (useNodeTwo) rootNode2IsLeaf else rootNode1IsLeaf
fun readBlockType() = TreeBlockType[reader.readASCIIString(2)]
fun findAllKeys(index: Long = rootNodeIndex): List<ByteArray> {
seekBlock(index)
val list = ArrayList<ByteArray>()
val type = readBlockType()
if (type == TreeBlockType.LEAF) {
val keyAmount = reader.readInt()
// offset внутри лепестка в байтах
var offset = 6
for (i in 0 until keyAmount) {
// читаем ключ
list.add(ByteArray(indexKeySize).also { reader.read(it) })
offset += indexKeySize
// читаем размер данных внутри ключа
var (dataLength, readBytes) = reader.readVarIntInfo()
offset += readBytes
while (true) {
// если конец данных внутри текущего блока, останавливаемся
if (offset + dataLength <= blockSize - 4) {
reader.skipBytes(dataLength)
offset += dataLength
break
}
// иначе, ищем следующий блок
// пропускаем оставшиеся данные, переходим на границу текущего блока-лепестка
val delta = (blockSize - 4 - offset).toInt()
reader.skipBytes(delta)
// ищем следующий блок с нашими данными
val nextBlockIndex = reader.readInt()
seekBlock(nextBlockIndex.toLong())
// удостоверяемся что мы попали в лепесток
check(readBlockType() == TreeBlockType.LEAF) { "Did not hit leaf block" }
offset = 2
dataLength -= delta
}
}
} else if (type == TreeBlockType.INDEX) {
reader.skipBytes(1)
val keyAmount = reader.readInt()
val blockList = IntArraySet()
blockList.add(reader.readInt())
for (i in 0 until keyAmount) {
// ключ
reader.skipBytes(indexKeySize)
// указатель на блок
blockList.add(reader.readInt())
}
// читаем все дочерние блоки на ключи
for (block in blockList.intIterator()) {
for (key in findAllKeys(block.toLong())) {
list.add(key)
}
}
}
return list
}
fun read(key: ByteArray): ByteArray? {
require(key.size == indexKeySize) { "Key provided is ${key.size} in size, while $indexKeySize is required" }
seekBlock(rootNodeIndex)
var type = readBlockType()
var iterations = 1000
val keyLoader = ByteArray(indexKeySize)
// сканирование индекса
while (iterations-- > 0 && type != TreeBlockType.LEAF) {
if (type == TreeBlockType.FREE) {
throw IllegalStateException("Hit free block while scanning index for ${key.joinToString(", ")}")
}
reader.skipBytes(1)
val keyCount = reader.readInt()
// if keyAmount == 4 then
// B a B b B c B d B
val readKeys = ByteArray((keyCount + 1) * 4 + keyCount * indexKeySize)
reader.readFully(readKeys)
val stream = DataInputStream(ByteArrayInputStream(readKeys))
var read = false
// B a
// B b
// B c
// B d
for (keyIndex in 0 until keyCount) {
// указатель на левый блок
val pointer = stream.readInt()
// левый ключ, всё что меньше него находится в левом блоке
stream.readFully(keyLoader)
// нужный ключ меньше самого первого ключа, поэтому он находится где то в левом блоке
if (key < keyLoader) {
seekBlock(pointer.toLong())
type = readBlockType()
read = true
break
}
}
if (!read) {
// ... B
seekBlock(stream.readInt().toLong())
type = readBlockType()
}
}
// мы пришли в лепесток, теперь прямолинейно ищем в linked list
var offset = 6
val keyCount = reader.readInt()
for (keyIndex in 0 until keyCount) {
// читаем ключ
reader.read(keyLoader)
offset += indexKeySize
// читаем размер данных
var (dataLength, readBytes) = reader.readVarIntInfo()
offset += readBytes
// это наш блок
if (keyLoader.contentEquals(key)) {
val binary = ByteArray(dataLength)
var binaryOffset = 0
// читаем данные
while (true) {
// если конец данных внутри текущего блока, останавливаемся
if (offset + dataLength <= blockSize - 4) {
reader.readFully(binary, binaryOffset, dataLength)
offset += dataLength
binaryOffset += dataLength
break
}
// иначе, ищем следующий блок
// пропускаем оставшиеся данные, переходим на границу текущего блока-лепестка
val delta = (blockSize - 4 - offset).toInt()
reader.readFully(binary, binaryOffset, delta)
binaryOffset += delta
// ищем следующий блок с нашими данными
val nextBlockIndex = reader.readInt()
seekBlock(nextBlockIndex.toLong())
// удостоверяемся что мы попали в лепесток
check(readBlockType() == TreeBlockType.LEAF) { "Did not hit leaf block" }
offset = 2
dataLength -= delta
}
return binary
} else {
// это не наш блок, пропускаем его
while (true) {
// если конец данных внутри текущего блока, останавливаемся
if (offset + dataLength <= blockSize - 4) {
reader.skipBytes(dataLength)
offset += dataLength
break
}
// иначе, ищем следующий блок
// пропускаем оставшиеся данные, переходим на границу текущего блока-лепестка
val delta = (blockSize - 4 - offset).toInt()
reader.skipBytes(delta)
// ищем следующий блок с нашими данными
val nextBlockIndex = reader.readInt()
seekBlock(nextBlockIndex.toLong())
// удостоверяемся что мы попали в лепесток
check(readBlockType() == TreeBlockType.LEAF) { "Did not hit leaf block" }
offset = 2
dataLength -= delta
}
}
}
return null
}
fun seekBlock(id: Long) {
require(id >= 0) { "Negative id $id" }
reader.seek(id * blockSize + blocksOffsetStart)
}
}

View File

@ -172,3 +172,20 @@ object BinaryJson {
return build
}
}
class VersionedJSON(var name: String = "Versioned JSON") {
var isVersioned = false
var version = 0
var data: JsonElement? = null
constructor(stream: DataInputStream) : this() {
name = stream.readASCIIString(stream.readVarInt())
isVersioned = stream.readBoolean()
if (isVersioned) {
version = stream.readInt()
}
data = BinaryJson.readElement(stream)
}
}

View File

@ -1,5 +1,6 @@
package ru.dbotthepony.kstarbound.io
import it.unimi.dsi.fastutil.bytes.ByteArrayList
import java.io.DataInputStream
import java.io.IOException
import java.io.InputStream
@ -25,6 +26,8 @@ fun RandomAccessFile.readVarLong(): Long {
return result
}
data class VarIntReadResult(val value: Int, val cells: Int)
/**
* Читает Variable Length Integer как Int
*/
@ -45,6 +48,28 @@ fun RandomAccessFile.readVarInt(): Int {
return result
}
/**
* Читает Variable Length Integer как Int
*/
fun RandomAccessFile.readVarIntInfo(): VarIntReadResult {
var result = 0
var read = read()
var i = 1
while (true) {
result = (result shl 7) or (read and 0x7F)
if (read and 0x80 == 0) {
break
}
read = read()
i++
}
return VarIntReadResult(result, i)
}
/**
* Читает Variable Length Integer как Long
*/
@ -112,3 +137,35 @@ fun InputStream.readASCIIString(length: Int): String {
return bytes.toString(Charsets.UTF_8)
}
fun RandomAccessFile.readCString(): String {
val bytes = ByteArrayList()
var read = read()
while (read != 0) {
bytes.add(read.toByte())
read = read()
}
return ByteArray(bytes.size).also {
for (i in it.indices) {
it[i] = bytes.getByte(i)
}
}.toString(Charsets.UTF_8)
}
fun InputStream.readCString(): String {
val bytes = ByteArrayList()
var read = read()
while (read != 0) {
bytes.add(read.toByte())
read = read()
}
return ByteArray(bytes.size).also {
for (i in it.indices) {
it[i] = bytes.getByte(i)
}
}.toString(Charsets.UTF_8)
}