btreedb reading test

src/main/kotlin/ru/dbotthepony/kstarbound/Main.kt

@@ -11,6 +11,7 @@ import ru.dbotthepony.kstarbound.client.StarboundClient
 import ru.dbotthepony.kstarbound.defs.TileDefinition
 import ru.dbotthepony.kstarbound.defs.projectile.ProjectilePhysics
 import ru.dbotthepony.kstarbound.defs.world.dungeon.DungeonWorldDef
+import ru.dbotthepony.kstarbound.io.*
 import ru.dbotthepony.kstarbound.world.CHUNK_SIZE_FF
 import ru.dbotthepony.kstarbound.world.Chunk
 import ru.dbotthepony.kstarbound.world.ChunkPos
@@ -18,8 +19,12 @@ import ru.dbotthepony.kstarbound.world.entities.Move
 import ru.dbotthepony.kstarbound.world.entities.PlayerEntity
 import ru.dbotthepony.kstarbound.world.entities.projectile.Projectile
 import ru.dbotthepony.kvector.vector.ndouble.Vector2d
+import java.io.ByteArrayInputStream
+import java.io.DataInputStream
 import java.io.File
+import java.io.InputStream
 import java.util.*
+import java.util.zip.Inflater
 
 private val LOGGER = LogManager.getLogger()
 
@@ -34,6 +39,52 @@ fun main() {
 		//return
 	}
 
+	val db = BTreeDB(File("F:\\SteamLibrary\\steamapps\\common\\Starbound - Unstable\\storage\\universe\\389760395_938904237_-238610574_5.world"))
+
+	/*if (true) {
+		val a = System.currentTimeMillis()
+		val worldMeta = db.read(byteArrayOf(0, 0, 0, 0, 0))
+		println(System.currentTimeMillis() - a)
+
+		val inflater = Inflater()
+		inflater.setInput(worldMeta!!)
+
+		val output = ByteArray(1_000_000)
+		inflater.inflate(output)
+
+		val stream = DataInputStream(ByteArrayInputStream(output))
+		println("X tiles ${stream.readInt()}")
+		println("Y tiles ${stream.readInt()}")
+
+		val metadata = VersionedJSON(stream)
+		println(metadata.data)
+
+		return
+	}*/
+
+	/*if (true) {
+		val data = db.read(byteArrayOf(1, 0, 61, 0, 23))
+
+		val inflater = Inflater()
+		inflater.setInput(data!!)
+
+		val output = ByteArray(64_000)
+		val actual = inflater.inflate(output)
+		File("F:\\SteamLibrary\\steamapps\\common\\Starbound - Unstable\\storage\\universe\\tiles.dat").writeBytes(output)
+		val reader = DataInputStream(ByteArrayInputStream(output))
+
+		reader.skipBytes(3)
+
+		for (y in 0 .. 31) {
+			for (x in 0 .. 31) {
+				println("$x $y ${reader.readShort()}")
+				reader.skipBytes(29)
+			}
+		}
+
+		return
+	}*/
+
 	val client = StarboundClient()
 
 	//Starbound.addFilePath(File("./unpacked_assets/"))
@@ -47,138 +98,61 @@ fun main() {
 		Starbound.terminateLoading = true
 	}
 
-	var chunkA: Chunk<*, *>? = null
-
 	val ent = PlayerEntity(client.world!!)
 
 	Starbound.onInitialize {
-		client.world!!.parallax = Starbound.parallaxAccess["barren"]
+		for (chunkX in 0 .. 61) {
-		chunkA = client.world!!.computeIfAbsent(ChunkPos(0, 0)).chunk
+			for (chunkY in 0 .. 61) {
-		val chunkB = client.world!!.computeIfAbsent(ChunkPos(-1, 0)).chunk
+				val data = db.read(byteArrayOf(1, 0, chunkX.toByte(), 0, chunkY.toByte()))
-		val chunkC = client.world!!.computeIfAbsent(ChunkPos(-2, 0)).chunk
 
-		val tile = Starbound.getTileDefinition("alienrock")
+				if (data != null) {
+					val chunk = client.world!!.computeIfAbsent(ChunkPos(chunkX, chunkY))
+					val inflater = Inflater()
+					inflater.setInput(data)
 
-		for (x in -6 .. 6) {
+					val output = ByteArray(64_000)
-			for (y in 0 .. 4) {
+					val actual = inflater.inflate(output)
-				val chnk = client.world!!.computeIfAbsent(ChunkPos(x, y))
+					val reader = DataInputStream(ByteArrayInputStream(output))
 
-				if (y == 0) {
+					reader.skipBytes(3)
-					for (bx in 0 .. 31) {
+
-						for (by in 0 .. 3) {
+					var hitTile = false
-							chnk.chunk.foreground[bx, by] = tile
+
+					for (y in 0 .. 31) {
+						for (x in 0 .. 31) {
+							val materialID = reader.readShort()
+							val getMat = Starbound.tilesAccessID[materialID.toInt()]
+
+							if (getMat != null) {
+								chunk.chunk.foreground[x, y] = getMat
+								hitTile = true
+							}
+
+							reader.skipBytes(5)
+
+							val materialID2 = reader.readShort()
+							val getMat2 = Starbound.tilesAccessID[materialID2.toInt()]
+
+							if (getMat2 != null) {
+								chunk.chunk.background[x, y] = getMat2
+								hitTile = true
+							}
+
+							reader.skipBytes(22)
 						}
 					}
-				}
-			}
-		}
 
-		run {
+					if (hitTile) {
-			val chunk = client.world!!.computeIfAbsent(ChunkPos(-2, 0)).chunk
+						//println(chunk.chunk.posVector2d)
-
+						// ent.position = chunk.chunk.posVector2d + Vector2d(16.0, 34.0)
-			for (y in 0 .. CHUNK_SIZE_FF) {
-				for (x in 0 .. (CHUNK_SIZE_FF - y)) {
-					chunk.foreground[x, y] = tile
-				}
-			}
-		}
-
-		run {
-			val chunk = client.world!!.computeIfAbsent(ChunkPos(-3, 0)).chunk
-
-			for (y in 0 .. CHUNK_SIZE_FF) {
-				for (x in 0 .. (CHUNK_SIZE_FF - y * 2)) {
-					chunk.foreground[x, y] = tile
-				}
-			}
-		}
-
-		for (x in 0 .. 31) {
-			for (y in 0 .. 3) {
-				chunkA!!.foreground[x, y] = tile
-			}
-		}
-
-		for (x in 0 .. 31) {
-			for (y in 8 .. 9) {
-				chunkA!!.foreground[x, y] = tile
-			}
-		}
-
-		for (x in 0 .. 31) {
-			for (y in 0 .. 0) {
-				chunkB.foreground[x, y] = tile
-			}
-		}
-
-		for (x in 4 .. 8) {
-			for (y in 4 .. 8) {
-				chunkA!!.foreground[x, y] = null as TileDefinition?
-			}
-		}
-
-		chunkA!!.foreground[18, 14] = tile
-
-		/*val rand = Random()
-
-		for (i in 0 .. 400) {
-			chunkA!!.foreground[rand.nextInt(0, CHUNK_SIZE_FF), rand.nextInt(0, CHUNK_SIZE_FF)] = tile
-		}*/
-
-		// ent.movement.dropToFloor()
-
-		run {
-			var i = 0
-
-			for (proj in Starbound.projectilesAccess.values) {
-				if (proj.physics == ProjectilePhysics.BOUNCY) {
-					val projEnt = Projectile(client.world!!, proj)
-					projEnt.position = Vector2d(i * 2.0, 18.0)
-					projEnt.spawn()
-					i++
-				}
-			}
-		}
-
-		for (i in 0 .. 10) {
-			client.world!!.timer(i * 1.0, 1) {
-				val projEnt = Projectile(client.world!!, Starbound.projectilesAccess["pill"]!!)
-				projEnt.position = Vector2d(i * 2.0 - 15.0, 13.0)
-				projEnt.spawn()
-			}
-		}
-
-		run {
-			val stripes = 0
-
-			for (stripe in 0 until stripes) {
-				for (x in 0 .. (stripes - stripe)) {
-					val movingBody = client.world!!.physics.createBody(BodyDef(
-						type = BodyType.DYNAMIC,
-						position = Vector2d(x = (-stripes + stripe) * 1.0 + x * 2.1, y = 8.0 + stripe * 2.1),
-						gravityScale = 1.1
-					))
-
-					val dynamicBox: IShape<*>
-
-					if (false) {
-						dynamicBox = PolygonShape()
-						dynamicBox.setAsBox(1.0, 1.0)
-					} else {
-						dynamicBox = CircleShape(1.0)
 					}
-
-					movingBody.createFixture(FixtureDef(
-						shape = dynamicBox,
-						density = 1.0,
-						friction = 0.3
-					))
 				}
 			}
 		}
 	}
 
-	ent.position += Vector2d(y = 14.0, x = -10.0)
+	//ent.position += Vector2d(y = 14.0, x = -10.0)
+	ent.position = Vector2d(128.0 + 16.0, 672.0 + 48.0)
 
 	client.onDrawGUI {
 		client.gl.font.render("${ent.position}", y = 100f, scale = 0.25f)

src/main/kotlin/ru/dbotthepony/kstarbound/Starbound.kt

@@ -1,6 +1,7 @@
 package ru.dbotthepony.kstarbound
 
 import com.google.gson.*
+import it.unimi.dsi.fastutil.ints.Int2ObjectAVLTreeMap
 import org.apache.logging.log4j.LogManager
 import ru.dbotthepony.kstarbound.api.IVFS
 import ru.dbotthepony.kstarbound.api.PhysicalFS
@@ -38,11 +39,13 @@ object Starbound : IVFS {
 	private val LOGGER = LogManager.getLogger()
 
 	private val tiles = HashMap<String, TileDefinition>()
+	private val tilesByMaterialID = Int2ObjectAVLTreeMap<TileDefinition>()
 	private val projectiles = HashMap<String, ConfiguredProjectile>()
 	private val parallax = HashMap<String, ParallaxPrototype>()
 	private val functions = HashMap<String, JsonFunction>()
 
 	val tilesAccess: Map<String, TileDefinition> = Collections.unmodifiableMap(tiles)
+	val tilesAccessID: Map<Int, TileDefinition> = Collections.unmodifiableMap(tilesByMaterialID)
 	val projectilesAccess: Map<String, ConfiguredProjectile> = Collections.unmodifiableMap(projectiles)
 	val parallaxAccess: Map<String, ParallaxPrototype> = Collections.unmodifiableMap(parallax)
 	val functionsAccess: Map<String, JsonFunction> = Collections.unmodifiableMap(functions)
@@ -230,7 +233,9 @@ object Starbound : IVFS {
 
 						val tileDef = TileDefinitionBuilder.fromJson(JsonParser.parseReader(getReader(listedFile)) as JsonObject).build("/tiles/materials")
 
-						check(tiles[tileDef.materialName] == null) { "Already has material with ID ${tileDef.materialName} loaded!" }
+						check(tiles[tileDef.materialName] == null) { "Already has material with name ${tileDef.materialName} loaded!" }
+						check(tilesByMaterialID[tileDef.materialId] == null) { "Already has material with ID ${tileDef.materialId} loaded!" }
+						tilesByMaterialID[tileDef.materialId] = tileDef
 						tiles[tileDef.materialName] = tileDef
 					} catch (err: Throwable) {
 						//throw TileDefLoadingException("Loading tile file $listedFile", err)

src/main/kotlin/ru/dbotthepony/kstarbound/io/BTreeDB.kt

@@ -0,0 +1,300 @@
+package ru.dbotthepony.kstarbound.io
+
+import com.google.gson.JsonElement
+import it.unimi.dsi.fastutil.ints.IntArraySet
+import java.io.ByteArrayInputStream
+import java.io.DataInputStream
+import java.io.File
+import java.io.InputStream
+import java.io.RandomAccessFile
+
+private fun readHeader(reader: RandomAccessFile, required: Char) {
+	val read = reader.read()
+	require(read.toChar() == required) { "Bad Starbound Pak header, expected ${required.code}, got $read" }
+}
+
+enum class TreeBlockType(val identity: String) {
+	INDEX("II"),
+	LEAF("LL"),
+	FREE("FF");
+
+	companion object {
+		operator fun get(index: String): TreeBlockType {
+			return when (index) {
+				INDEX.identity -> INDEX
+				LEAF.identity -> LEAF
+				FREE.identity -> FREE
+				else -> throw NoSuchElementException("Unknown block type $index")
+			}
+		}
+	}
+}
+
+private operator fun ByteArray.compareTo(b: ByteArray): Int {
+	require(size == b.size) { "Keys are not of same size (${size} vs ${b.size})" }
+
+	for (i in indices) {
+		if (this[i] > b[i]) {
+			return 1
+		} else if (this[i] < b[i]) {
+			return -1
+		}
+	}
+
+	return 0
+}
+
+/**
+ * Класс, который позволяет читать и записывать в файлы Starbound BTReeDB 5
+ *
+ * Big credit for https://github.com/blixt/py-starbound/blob/master/FORMATS.md#btreedb5 !
+ */
+class BTreeDB(val path: File) {
+	val reader = RandomAccessFile(path, "r")
+
+	init {
+		readHeader(reader, 'B')
+		readHeader(reader, 'T')
+		readHeader(reader, 'r')
+		readHeader(reader, 'e')
+		readHeader(reader, 'e')
+		readHeader(reader, 'D')
+		readHeader(reader, 'B')
+		readHeader(reader, '5')
+	}
+
+	val blockSize = reader.readInt().toLong()
+	val dbNameRaw = ByteArray(16).also { reader.read(it) }
+	val indexKeySize = reader.readInt()
+	val useNodeTwo = reader.readBoolean()
+	val freeNodeIndex1 = reader.readInt().toLong()
+
+	init { reader.skipBytes(4) }
+	val freeBlockOffset1 = reader.readInt().toLong()
+	val rootNode1Index = reader.readInt().toLong()
+	val rootNode1IsLeaf = reader.readBoolean()
+	val freeNodeIndex2 = reader.readInt().toLong()
+	init { reader.skipBytes(4) }
+	val freeBlockOffset2 = reader.readInt().toLong()
+	val rootNode2Index = reader.readInt().toLong()
+	val rootNode2IsLeaf = reader.readBoolean()
+	init { reader.skipBytes(445) }
+
+	val blocksOffsetStart = reader.filePointer
+
+	init {
+		// check(reader.length() - blocksOffsetStart == 512L) { "Unexpected header size of ${reader.length() - blocksOffsetStart} bytes" }
+		check((reader.length() - 512L) % blockSize == 0L) { "Junk data somewhere in file (${(reader.length() - 512L) % blockSize} lingering bytes)" }
+	}
+
+	val rootNodeIndex get() = if (useNodeTwo) rootNode2Index else rootNode1Index
+	val rootNodeIsLeaf get() = if (useNodeTwo) rootNode2IsLeaf else rootNode1IsLeaf
+
+	fun readBlockType() = TreeBlockType[reader.readASCIIString(2)]
+
+	fun findAllKeys(index: Long = rootNodeIndex): List<ByteArray> {
+		seekBlock(index)
+
+		val list = ArrayList<ByteArray>()
+		val type = readBlockType()
+
+		if (type == TreeBlockType.LEAF) {
+			val keyAmount = reader.readInt()
+			// offset внутри лепестка в байтах
+			var offset = 6
+
+			for (i in 0 until keyAmount) {
+				// читаем ключ
+				list.add(ByteArray(indexKeySize).also { reader.read(it) })
+				offset += indexKeySize
+
+				// читаем размер данных внутри ключа
+				var (dataLength, readBytes) = reader.readVarIntInfo()
+				offset += readBytes
+
+				while (true) {
+					// если конец данных внутри текущего блока, останавливаемся
+					if (offset + dataLength <= blockSize - 4) {
+						reader.skipBytes(dataLength)
+						offset += dataLength
+						break
+					}
+
+					// иначе, ищем следующий блок
+
+					// пропускаем оставшиеся данные, переходим на границу текущего блока-лепестка
+					val delta = (blockSize - 4 - offset).toInt()
+					reader.skipBytes(delta)
+
+					// ищем следующий блок с нашими данными
+					val nextBlockIndex = reader.readInt()
+					seekBlock(nextBlockIndex.toLong())
+
+					// удостоверяемся что мы попали в лепесток
+					check(readBlockType() == TreeBlockType.LEAF) { "Did not hit leaf block" }
+					offset = 2
+					dataLength -= delta
+				}
+			}
+		} else if (type == TreeBlockType.INDEX) {
+			reader.skipBytes(1)
+			val keyAmount = reader.readInt()
+
+			val blockList = IntArraySet()
+			blockList.add(reader.readInt())
+
+			for (i in 0 until keyAmount) {
+				// ключ
+				reader.skipBytes(indexKeySize)
+
+				// указатель на блок
+				blockList.add(reader.readInt())
+			}
+
+			// читаем все дочерние блоки на ключи
+			for (block in blockList.intIterator()) {
+				for (key in findAllKeys(block.toLong())) {
+					list.add(key)
+				}
+			}
+		}
+
+		return list
+	}
+
+	fun read(key: ByteArray): ByteArray? {
+		require(key.size == indexKeySize) { "Key provided is ${key.size} in size, while $indexKeySize is required" }
+
+		seekBlock(rootNodeIndex)
+		var type = readBlockType()
+		var iterations = 1000
+
+		val keyLoader = ByteArray(indexKeySize)
+
+		// сканирование индекса
+		while (iterations-- > 0 && type != TreeBlockType.LEAF) {
+			if (type == TreeBlockType.FREE) {
+				throw IllegalStateException("Hit free block while scanning index for ${key.joinToString(", ")}")
+			}
+
+			reader.skipBytes(1)
+
+			val keyCount = reader.readInt()
+			// if keyAmount == 4 then
+			// B a B b B c B d B
+			val readKeys = ByteArray((keyCount + 1) * 4 + keyCount * indexKeySize)
+			reader.readFully(readKeys)
+
+			val stream = DataInputStream(ByteArrayInputStream(readKeys))
+
+			var read = false
+
+			// B a
+			// B b
+			// B c
+			// B d
+			for (keyIndex in 0 until keyCount) {
+				// указатель на левый блок
+				val pointer = stream.readInt()
+
+				// левый ключ, всё что меньше него находится в левом блоке
+				stream.readFully(keyLoader)
+
+				// нужный ключ меньше самого первого ключа, поэтому он находится где то в левом блоке
+				if (key < keyLoader) {
+					seekBlock(pointer.toLong())
+					type = readBlockType()
+					read = true
+					break
+				}
+			}
+
+			if (!read) {
+				// ... B
+				seekBlock(stream.readInt().toLong())
+				type = readBlockType()
+			}
+		}
+
+		// мы пришли в лепесток, теперь прямолинейно ищем в linked list
+		var offset = 6
+		val keyCount = reader.readInt()
+
+		for (keyIndex in 0 until keyCount) {
+			// читаем ключ
+			reader.read(keyLoader)
+			offset += indexKeySize
+
+			// читаем размер данных
+			var (dataLength, readBytes) = reader.readVarIntInfo()
+			offset += readBytes
+
+			// это наш блок
+			if (keyLoader.contentEquals(key)) {
+				val binary = ByteArray(dataLength)
+				var binaryOffset = 0
+
+				// читаем данные
+				while (true) {
+					// если конец данных внутри текущего блока, останавливаемся
+					if (offset + dataLength <= blockSize - 4) {
+						reader.readFully(binary, binaryOffset, dataLength)
+						offset += dataLength
+						binaryOffset += dataLength
+						break
+					}
+
+					// иначе, ищем следующий блок
+
+					// пропускаем оставшиеся данные, переходим на границу текущего блока-лепестка
+					val delta = (blockSize - 4 - offset).toInt()
+					reader.readFully(binary, binaryOffset, delta)
+					binaryOffset += delta
+
+					// ищем следующий блок с нашими данными
+					val nextBlockIndex = reader.readInt()
+					seekBlock(nextBlockIndex.toLong())
+
+					// удостоверяемся что мы попали в лепесток
+					check(readBlockType() == TreeBlockType.LEAF) { "Did not hit leaf block" }
+					offset = 2
+					dataLength -= delta
+				}
+
+				return binary
+			} else {
+				// это не наш блок, пропускаем его
+				while (true) {
+					// если конец данных внутри текущего блока, останавливаемся
+					if (offset + dataLength <= blockSize - 4) {
+						reader.skipBytes(dataLength)
+						offset += dataLength
+						break
+					}
+
+					// иначе, ищем следующий блок
+
+					// пропускаем оставшиеся данные, переходим на границу текущего блока-лепестка
+					val delta = (blockSize - 4 - offset).toInt()
+					reader.skipBytes(delta)
+
+					// ищем следующий блок с нашими данными
+					val nextBlockIndex = reader.readInt()
+					seekBlock(nextBlockIndex.toLong())
+
+					// удостоверяемся что мы попали в лепесток
+					check(readBlockType() == TreeBlockType.LEAF) { "Did not hit leaf block" }
+					offset = 2
+					dataLength -= delta
+				}
+			}
+		}
+
+		return null
+	}
+
+	fun seekBlock(id: Long) {
+		require(id >= 0) { "Negative id $id" }
+		reader.seek(id * blockSize + blocksOffsetStart)
+	}
+}

src/main/kotlin/ru/dbotthepony/kstarbound/io/BinaryJson.kt

@@ -172,3 +172,20 @@ object BinaryJson {
 		return build
 	}
 }
+
+class VersionedJSON(var name: String = "Versioned JSON") {
+	var isVersioned = false
+	var version = 0
+	var data: JsonElement? = null
+
+	constructor(stream: DataInputStream) : this() {
+		name = stream.readASCIIString(stream.readVarInt())
+		isVersioned = stream.readBoolean()
+
+		if (isVersioned) {
+			version = stream.readInt()
+		}
+
+		data = BinaryJson.readElement(stream)
+	}
+}

src/main/kotlin/ru/dbotthepony/kstarbound/io/Ext.kt

@@ -1,5 +1,6 @@
 package ru.dbotthepony.kstarbound.io
 
+import it.unimi.dsi.fastutil.bytes.ByteArrayList
 import java.io.DataInputStream
 import java.io.IOException
 import java.io.InputStream
@@ -25,6 +26,8 @@ fun RandomAccessFile.readVarLong(): Long {
 	return result
 }
 
+data class VarIntReadResult(val value: Int, val cells: Int)
+
 /**
  * Читает Variable Length Integer как Int
  */
@@ -45,6 +48,28 @@ fun RandomAccessFile.readVarInt(): Int {
 	return result
 }
 
+/**
+ * Читает Variable Length Integer как Int
+ */
+fun RandomAccessFile.readVarIntInfo(): VarIntReadResult {
+	var result = 0
+	var read = read()
+	var i = 1
+
+	while (true) {
+		result = (result shl 7) or (read and 0x7F)
+
+		if (read and 0x80 == 0) {
+			break
+		}
+
+		read = read()
+		i++
+	}
+
+	return VarIntReadResult(result, i)
+}
+
 /**
  * Читает Variable Length Integer как Long
  */
@@ -112,3 +137,35 @@ fun InputStream.readASCIIString(length: Int): String {
 
 	return bytes.toString(Charsets.UTF_8)
 }
+
+fun RandomAccessFile.readCString(): String {
+	val bytes = ByteArrayList()
+	var read = read()
+
+	while (read != 0) {
+		bytes.add(read.toByte())
+		read = read()
+	}
+
+	return ByteArray(bytes.size).also {
+		for (i in it.indices) {
+			it[i] = bytes.getByte(i)
+		}
+	}.toString(Charsets.UTF_8)
+}
+
+fun InputStream.readCString(): String {
+	val bytes = ByteArrayList()
+	var read = read()
+
+	while (read != 0) {
+		bytes.add(read.toByte())
+		read = read()
+	}
+
+	return ByteArray(bytes.size).also {
+		for (i in it.indices) {
+			it[i] = bytes.getByte(i)
+		}
+	}.toString(Charsets.UTF_8)
+}