Starbound Pack File!

This commit is contained in:
DBotThePony 2022-02-08 22:26:20 +07:00
parent a8af98f715
commit 70772344e6
Signed by: DBot
GPG Key ID: DCC23B5715498507
12 changed files with 568 additions and 61 deletions

View File

@ -58,6 +58,7 @@ dependencies {
implementation("org.lwjgl", "lwjgl-par")
implementation("org.lwjgl", "lwjgl-stb")
implementation("org.lwjgl", "lwjgl-vulkan")
runtimeOnly("org.lwjgl", "lwjgl", classifier = lwjglNatives)
runtimeOnly("org.lwjgl", "lwjgl-assimp", classifier = lwjglNatives)
runtimeOnly("org.lwjgl", "lwjgl-bgfx", classifier = lwjglNatives)

View File

@ -3,10 +3,10 @@ package ru.dbotthepony.kstarbound
import org.apache.logging.log4j.LogManager
import org.lwjgl.Version
import org.lwjgl.glfw.GLFW.glfwSetWindowShouldClose
import ru.dbotthepony.kstarbound.api.PhysicalFS
import ru.dbotthepony.kstarbound.client.StarboundClient
import ru.dbotthepony.kstarbound.defs.TileDefinition
import ru.dbotthepony.kstarbound.lua.LuaJNI
import ru.dbotthepony.kstarbound.lua.LuaState
import ru.dbotthepony.kstarbound.io.StarboundPak
import ru.dbotthepony.kstarbound.math.Vector2d
import ru.dbotthepony.kstarbound.world.CHUNK_SIZE_FF
import ru.dbotthepony.kstarbound.world.Chunk
@ -22,7 +22,8 @@ fun main() {
val client = StarboundClient()
Starbound.addFilePath(File("./unpacked_assets/"))
//Starbound.addFilePath(File("./unpacked_assets/"))
Starbound.addPakPath(File("J:\\SteamLibrary\\steamapps\\common\\Starbound\\assets\\packed.pak"))
Starbound.initializeGame { finished, replaceStatus, status ->
client.putDebugLog(status, replaceStatus)

View File

@ -3,11 +3,17 @@ package ru.dbotthepony.kstarbound
import com.google.gson.JsonElement
import com.google.gson.JsonObject
import com.google.gson.JsonParser
import ru.dbotthepony.kstarbound.api.IVFS
import ru.dbotthepony.kstarbound.api.PhysicalFS
import ru.dbotthepony.kstarbound.defs.TileDefinition
import ru.dbotthepony.kstarbound.defs.TileDefinitionBuilder
import ru.dbotthepony.kstarbound.io.StarboundPak
import ru.dbotthepony.kstarbound.world.World
import java.io.File
import java.io.FileNotFoundException
import java.io.*
import java.nio.ByteBuffer
import java.util.*
import kotlin.collections.ArrayList
import kotlin.collections.HashMap
const val METRES_IN_STARBOUND_UNIT = 0.25
const val METRES_IN_STARBOUND_UNITf = 0.25f
@ -17,7 +23,7 @@ const val PIXELS_IN_STARBOUND_UNITf = 8.0f
class TileDefLoadingException(message: String, cause: Throwable? = null) : IllegalStateException(message, cause)
object Starbound {
object Starbound : IVFS {
private val tiles = HashMap<String, TileDefinition>()
val tilesAccess = object : Map<String, TileDefinition> by tiles {}
@ -27,42 +33,32 @@ object Starbound {
private set
var terminateLoading = false
private val _filepath = ArrayList<File>()
val filepath = object : List<File> by _filepath {}
private val archivePaths = ArrayList<File>()
private val fileSystems = ArrayList<IVFS>()
fun addFilePath(path: File) {
_filepath.add(path)
fileSystems.add(PhysicalFS(path))
}
fun findFile(path: File): File {
if (path.exists()) {
return path.canonicalFile
/**
* Добавляет уже прочитанный pak
*/
fun addPak(pak: StarboundPak) {
fileSystems.add(pak)
}
for (sPath in _filepath) {
val newPath = File(sPath.path, path.path)
if (newPath.exists()) {
return newPath
/**
* Добавляет pak к чтению при initializeGame
*/
fun addPakPath(pak: File) {
archivePaths.add(pak)
}
}
throw FileNotFoundException("Unable to find $path in any of known file paths")
}
fun findFile(path: String) = findFile(File(path))
fun loadJson(path: String): JsonElement {
if (path[0] == '/')
return JsonParser.parseReader(findFile(path.substring(1)).bufferedReader())
return JsonParser.parseReader(findFile(path).bufferedReader())
}
fun getTileDefinition(name: String): TileDefinition? {
return tiles[name]
return JsonParser.parseReader(getReader(path))
}
fun getTileDefinition(name: String) = tiles[name]
private val initCallbacks = ArrayList<() -> Unit>()
fun initializeGame(callback: (finished: Boolean, replaceStatus: Boolean, status: String) -> Unit) {
@ -78,6 +74,19 @@ object Starbound {
Thread({
val time = System.currentTimeMillis()
if (archivePaths.isNotEmpty()) {
callback(false, false, "Reading pak archives...")
for (path in archivePaths) {
callback(false, false, "Reading ${path.name}...")
addPak(StarboundPak(path) { _, status ->
callback(false, true, "${path.name}: $status")
})
}
}
callback(false, false, "Loading materials...")
loadTileMaterials {
@ -96,6 +105,36 @@ object Starbound {
}, "Asset Loader").start()
}
override fun pathExists(path: String): Boolean {
for (fs in fileSystems) {
if (fs.pathExists(path)) {
return true
}
}
return false
}
override fun readOrNull(path: String): ByteBuffer? {
for (fs in fileSystems) {
if (fs.pathExists(path)) {
return fs.read(path)
}
}
return null
}
override fun listFiles(path: String): List<String> {
val listing = mutableListOf<String>()
for (fs in fileSystems) {
listing.addAll(fs.listFiles(path))
}
return listing
}
fun onInitialize(callback: () -> Unit) {
if (initialized) {
callback()
@ -115,23 +154,18 @@ object Starbound {
}
private fun loadTileMaterials(callback: (String) -> Unit) {
for (sPath in _filepath) {
val newPath = File(sPath.path, "tiles/materials")
if (newPath.exists() && newPath.isDirectory) {
val findFiles = newPath.listFiles()!!
for (listedFile in findFiles) {
if (listedFile.path.endsWith(".material")) {
for (fs in fileSystems) {
for (listedFile in fs.listFiles("tiles/materials")) {
if (listedFile.endsWith(".material")) {
try {
callback("Loading ${listedFile.name}")
val tileDef = TileDefinitionBuilder.fromJson(JsonParser.parseReader(listedFile.bufferedReader()) as JsonObject).build(listedFile.parent)
callback("Loading $listedFile")
val tileDef = TileDefinitionBuilder.fromJson(JsonParser.parseReader(getReader(listedFile)) as JsonObject).build("/tiles/materials")
check(tiles[tileDef.materialName] == null) { "Already has material with ID ${tileDef.materialName} loaded!" }
tiles[tileDef.materialName] = tileDef
} catch (err: Throwable) {
throw TileDefLoadingException("Loading tile file ${listedFile.name}", err)
}
throw TileDefLoadingException("Loading tile file $listedFile", err)
}
}
}

View File

@ -0,0 +1,79 @@
package ru.dbotthepony.kstarbound.api
import java.io.*
import java.nio.ByteBuffer
interface IVFS {
fun pathExists(path: String): Boolean
fun read(path: String): ByteBuffer {
return readOrNull(path) ?: throw FileNotFoundException("No such file $path")
}
fun readOrNull(path: String): ByteBuffer?
fun readString(path: String): String {
return read(path).array().toString(Charsets.UTF_8)
}
fun getReader(path: String): Reader {
return InputStreamReader(ByteArrayInputStream(read(path).array()))
}
fun listFiles(path: String): Collection<String>
fun readDirect(path: String): ByteBuffer {
val read = read(path)
val buf = ByteBuffer.allocateDirect(read.capacity())
read.position(0)
for (i in 0 until read.capacity()) {
buf.put(read[i])
}
buf.position(0)
return buf
}
}
class PhysicalFS(root: File) : IVFS {
val root: File = root.absoluteFile
override fun pathExists(path: String): Boolean {
if (path.contains("..")) {
return false
}
return File(root.absolutePath, path).exists()
}
override fun readOrNull(path: String): ByteBuffer? {
if (path.contains("..")) {
return null
}
val fpath = File(root.path, path)
if (!fpath.exists()) {
return null
}
val reader = fpath.inputStream()
val buf = ByteBuffer.allocate(reader.channel.size().toInt())
reader.read(buf.array())
return buf
}
override fun listFiles(path: String): List<String> {
if (path.contains("..")) {
return listOf()
}
val fpath = File(root.absolutePath, path)
return fpath.listFiles()?.map {
it.path.replace('\\', '/').substring(root.path.length)
} ?: return listOf()
}
}

View File

@ -9,6 +9,7 @@ import org.lwjgl.system.MemoryStack
import org.lwjgl.system.MemoryUtil
import ru.dbotthepony.kstarbound.PIXELS_IN_STARBOUND_UNIT
import ru.dbotthepony.kstarbound.PIXELS_IN_STARBOUND_UNITf
import ru.dbotthepony.kstarbound.Starbound
import ru.dbotthepony.kstarbound.client.gl.GLStateTracker
import ru.dbotthepony.kstarbound.client.input.UserInput
import ru.dbotthepony.kstarbound.math.Matrix4f
@ -192,7 +193,7 @@ class StarboundClient : AutoCloseable {
val measure = GLFW.glfwGetTime()
if (frameRenderTime != 0.0)
if (frameRenderTime != 0.0 && Starbound.initialized)
world?.think(frameRenderTime)
glClear(GL_COLOR_BUFFER_BIT or GL_DEPTH_BUFFER_BIT)

View File

@ -13,6 +13,7 @@ import ru.dbotthepony.kstarbound.client.render.TileRenderers
import ru.dbotthepony.kstarbound.math.AABB
import ru.dbotthepony.kstarbound.util.Color
import java.io.File
import java.io.FileNotFoundException
import java.lang.ref.Cleaner
import java.util.concurrent.ThreadFactory
import kotlin.reflect.KProperty
@ -251,11 +252,13 @@ class GLStateTracker {
private val named2DTextures = HashMap<String, GLTexture2D>()
fun loadNamedTexture(path: File, memoryFormat: Int = GL_RGBA, fileFormat: Int = GL_RGBA): GLTexture2D {
val found = Starbound.findFile(path)
fun loadNamedTexture(path: String, memoryFormat: Int = GL_RGBA, fileFormat: Int = GL_RGBA): GLTexture2D {
if (!Starbound.pathExists(path)) {
throw FileNotFoundException("Unable to locate $path")
}
return named2DTextures.computeIfAbsent(found.absolutePath) {
return@computeIfAbsent newTexture(found.absolutePath).upload(found, memoryFormat, fileFormat).generateMips()
return named2DTextures.computeIfAbsent(path) {
return@computeIfAbsent newTexture(path).upload(Starbound.readDirect(path), memoryFormat, fileFormat).generateMips()
}
}

View File

@ -1,7 +1,6 @@
package ru.dbotthepony.kstarbound.client.gl
import org.apache.logging.log4j.LogManager
import org.lwjgl.opengl.GL11
import org.lwjgl.opengl.GL46.*
import org.lwjgl.stb.STBImage
import ru.dbotthepony.kstarbound.math.Vector2i
@ -147,6 +146,24 @@ class GLTexture2D(val state: GLStateTracker, val name: String = "<unknown>") : A
return this
}
fun upload(buff: ByteBuffer, memoryFormat: Int, bufferFormat: Int): GLTexture2D {
state.ensureSameThread()
val getwidth = intArrayOf(0)
val getheight = intArrayOf(0)
val getchannels = intArrayOf(0)
val bytes = STBImage.stbi_load_from_memory(buff, getwidth, getheight, getchannels, 0) ?: throw TextureLoadingException("Unable to load ${buff}. Is it a valid image?")
require(getwidth[0] > 0) { "Image $name has bad width of ${getwidth[0]}" }
require(getheight[0] > 0) { "Image $name has bad height of ${getheight[0]}" }
upload(memoryFormat, getwidth[0], getheight[0], bufferFormat, GL_UNSIGNED_BYTE, bytes)
STBImage.stbi_image_free(bytes)
return this
}
var isValid = true
private set

View File

@ -53,7 +53,7 @@ enum class TextAlignY {
class Font(
val state: GLStateTracker,
val font: String = "./unpacked_assets/hobo.ttf",
val font: String = "hobo.ttf",
val size: Int = 48
) {
val face = state.freeType.Face(font, 0L)

View File

@ -143,7 +143,7 @@ class TileDefinitionBuilder {
*/
data class TileRenderPiece(
val name: String,
val texture: File?,
val texture: String?,
val textureSize: Vector2i,
val texturePosition: Vector2i,
@ -157,7 +157,7 @@ data class TileRenderPiece(
throw UnsupportedOperationException("Render piece has not absolute texture path: $it")
}
return@let File(it.substring(1))
return@let it
}
val textureSize = Vector2i.fromJson(input["textureSize"].asJsonArray)
@ -526,7 +526,7 @@ data class TileRenderTemplate(
}
data class TileRenderDefinition(
val texture: File,
val texture: String,
val variants: Int,
val lightTransparent: Boolean,
val occludesBelow: Boolean,
@ -545,16 +545,16 @@ class TileRenderDefinitionBuilder {
var renderTemplate: TileRenderTemplate? = null
fun build(directory: String? = null): TileRenderDefinition {
val newtexture: File
val newtexture: String
if (texture[0] == '/') {
// путь абсолютен
newtexture = File(texture)
newtexture = texture
} else {
if (directory != null) {
newtexture = File(directory, texture)
newtexture = "$directory/$texture"
} else {
newtexture = File(texture)
newtexture = texture
}
}

View File

@ -0,0 +1,87 @@
package ru.dbotthepony.kstarbound.io
import com.google.gson.*
import java.io.RandomAccessFile
/**
* Интерфейс для чтения и записи двоичного формата Json от Chucklefish
*/
object BinaryJson {
const val TYPE_NULL = 0x01
const val TYPE_FLOAT = 0x02
const val TYPE_DOUBLE = TYPE_FLOAT
const val TYPE_BOOLEAN = 0x03
/**
* На самом деле, variable int
*/
const val TYPE_INT = 0x04
const val TYPE_STRING = 0x05
const val TYPE_ARRAY = 0x06
const val TYPE_OBJECT = 0x07
fun readElement(reader: RandomAccessFile): JsonElement {
return when (val id = reader.read()) {
TYPE_NULL -> JsonNull.INSTANCE
TYPE_DOUBLE -> JsonPrimitive(reader.readDouble())
TYPE_BOOLEAN -> JsonPrimitive(reader.readBoolean())
TYPE_INT -> JsonPrimitive(reader.readVarLong())
TYPE_STRING -> JsonPrimitive(reader.readASCIIString(reader.readVarInt()))
TYPE_ARRAY -> readArray(reader)
TYPE_OBJECT -> readObject(reader)
else -> throw JsonParseException("Unknown element type $id")
}
}
fun readObject(reader: RandomAccessFile): JsonObject {
val values = reader.readVarInt() - 1
if (values == -1) {
return JsonObject()
}
if (values < -1) {
throw JsonParseException("Tried to read json object with $values elements in it")
}
val build = JsonObject()
for (i in 0 .. values) {
val key: String
try {
key = reader.readASCIIString(reader.readVarInt())
} catch(err: Throwable) {
throw JsonParseException("Reading json object at $i", err)
}
try {
build.add(key, readElement(reader))
} catch(err: Throwable) {
throw JsonParseException("Reading json object at $i with name $key", err)
}
}
return build
}
fun readArray(reader: RandomAccessFile): JsonArray {
val values = reader.readVarInt() - 1
if (values == -1) {
return JsonArray()
}
if (values < -1) {
throw JsonParseException("Tried to read json array with $values elements in it")
}
val build = JsonArray()
for (i in 0 .. values) {
build.add(readElement(reader))
}
return build
}
}

View File

@ -0,0 +1,73 @@
package ru.dbotthepony.kstarbound.io
import java.io.IOException
import java.io.InputStream
import java.io.RandomAccessFile
/**
* Читает Variable Length Integer как Long
*/
fun RandomAccessFile.readVarLong(): Long {
var result = 0L
var read = read()
while (true) {
result = (result shl 7) or (read.toLong() and 0x7FL)
if (read and 0x80 == 0) {
break
}
read = read()
}
return result
}
/**
* Читает Variable Length Integer как Int
*/
fun RandomAccessFile.readVarInt(): Int {
var result = 0
var read = read()
while (true) {
result = (result shl 7) or (read and 0x7F)
if (read and 0x80 == 0) {
break
}
read = read()
}
return result
}
fun RandomAccessFile.readASCIIString(length: Int): String {
require(length >= 0) { "Invalid length $length" }
val bytes = ByteArray(length)
try {
readFully(bytes)
} catch(err: Throwable) {
throw IOException("Tried to read string with length of $length", err)
}
return bytes.toString(Charsets.UTF_8)
}
fun InputStream.readASCIIString(length: Int): String {
require(length >= 0) { "Invalid length $length" }
val bytes = ByteArray(length)
try {
val read = read(bytes)
require(read == bytes.size) { "Read $read, expected ${bytes.size}" }
} catch(err: Throwable) {
throw IOException("Tried to read string with length of $length", err)
}
return bytes.toString(Charsets.UTF_8)
}

View File

@ -0,0 +1,211 @@
package ru.dbotthepony.kstarbound.io
import ru.dbotthepony.kstarbound.api.IVFS
import java.io.*
import java.nio.ByteBuffer
import java.nio.channels.Channels
import java.util.*
import kotlin.collections.HashMap
private fun readHeader(reader: RandomAccessFile, required: Int) {
val read = reader.read()
require(read == required) { "Bad Starbound Pak header, expected $required, got $read" }
}
class StarboundPakFile(
val storage: StarboundPak,
val name: String,
val offset: Long,
val length: Long
) {
val directoryName: String
val directoryHiearchy: Array<String>
val fileName: String
init {
val split = name.substring(1).split('/').toMutableList()
fileName = split.last()
split.removeAt(split.size - 1)
directoryHiearchy = split.toTypedArray()
directoryName = split.joinToString("/")
}
fun read(): ByteBuffer {
val buf = ByteBuffer.allocate(length.toInt())
storage.reader.seek(offset)
storage.reader.readFully(buf.array())
return buf
}
override fun toString(): String {
return name
}
fun readString(): String {
return read().array().toString(Charsets.UTF_8)
}
companion object {
fun read(storage: StarboundPak, reader: RandomAccessFile): StarboundPakFile {
val readLength = reader.read()
//val name = reader.readASCIIString(readLength).lowercase()
val name = reader.readASCIIString(readLength)
require(name[0] == '/') { "$name appears to be not an absolute filename" }
val offset = reader.readLong()
val length = reader.readLong()
return StarboundPakFile(storage, name.intern(), offset, length)
}
fun read(storage: StarboundPak, reader: DataInputStream): StarboundPakFile {
val readLength = reader.read()
val name = reader.readASCIIString(readLength)
require(name[0] == '/') { "$name appears to be not an absolute filename" }
val offset = reader.readLong()
val length = reader.readLong()
return StarboundPakFile(storage, name.intern(), offset, length)
}
}
}
class StarboundPakDirectory(val name: String, val parent: StarboundPakDirectory? = null) {
private val files = HashMap<String, StarboundPakFile>()
private val directories = HashMap<String, StarboundPakDirectory>()
fun resolve(path: Array<String>, level: Int = 0): StarboundPakDirectory {
if (path.size == level) {
return this
}
if (level == 0 && path[0] == "" && name == "/")
return resolve(path, 1)
if (directories.containsKey(path[level])) {
return directories[path[level]]!!.resolve(path, level + 1)
} else {
val dir = StarboundPakDirectory(path[level], this)
directories[path[level]] = dir
return dir.resolve(path, level + 1)
}
}
fun getFile(name: String) = files[name]
fun getDirectory(name: String) = directories[name]
fun listFiles(): Collection<StarboundPakFile> = Collections.unmodifiableCollection(files.values)
fun writeFile(file: StarboundPakFile) {
files[file.name.split('/').last()] = file
}
fun fullName(): String {
var build = name
var getParent = parent
while (getParent != null) {
if (getParent.parent != null) {
build = "${getParent.name}/$name"
} else {
build = "/$name"
break
}
getParent = getParent.parent
}
return build
}
override fun toString(): String {
return fullName()
}
}
class StarboundPak(val path: File, callback: (finished: Boolean, status: String) -> Unit = { _, _ -> }) : Closeable, IVFS {
val reader = RandomAccessFile(path, "r")
init {
readHeader(reader, 0x53) // S
readHeader(reader, 0x42) // B
readHeader(reader, 0x41) // A
readHeader(reader, 0x73) // s
readHeader(reader, 0x73) // s
readHeader(reader, 0x65) // e
readHeader(reader, 0x74) // t
readHeader(reader, 0x36) // 6
}
// Далее идёт 8 байтный long в формате Big Endian, который указывает на offset до INDEX
// т.е. сделав seek(indexOffset) мы выйдем прямо на INDEX
private val indexOffset = reader.readLong()
init {
reader.seek(indexOffset)
readHeader(reader, 0x49) // I
readHeader(reader, 0x4E) // N
readHeader(reader, 0x44) // D
readHeader(reader, 0x45) // E
readHeader(reader, 0x58) // X
callback(false, "Reading metadata")
}
// сразу за INDEX идут метаданные в формате Binary Json
val metadata = BinaryJson.readObject(reader)
// сразу за метаданными идёт количество файлов внутри данного pak в формате Big Endian variable int
val indexNodeCount = reader.readVarLong()
private val indexNodes = HashMap<String, StarboundPakFile>()
val root = StarboundPakDirectory("/")
init {
// Сразу же за количеством файлов идут сами файлы в формате
// byte (длинна имени файла)
// byte[] (utf-8 имя файла)
// long (offset от начала файла)
// long (длинна файла)
val stream = DataInputStream(BufferedInputStream(Channels.newInputStream(reader.channel)))
for (i in 0 until indexNodeCount) {
try {
callback(false, "Reading index node $i")
val read = StarboundPakFile.read(this, stream)
if (read.offset > reader.length()) {
throw IndexOutOfBoundsException("Garbage offset at index $i: ${read.offset}")
}
if (read.length > reader.length()) {
throw IndexOutOfBoundsException("Garbage length at index $i: ${read.length}")
}
indexNodes[read.name] = read
root.resolve(read.directoryHiearchy).writeFile(read)
} catch (err: Throwable) {
throw IOException("Reading index node at $i", err)
}
}
callback(true, "Reading indexes finished")
}
override fun listFiles(path: String): Collection<String> {
return root.resolve(path.split("/").toTypedArray()).listFiles().map { it.name }
}
override fun pathExists(path: String): Boolean {
return indexNodes.containsKey(path)
}
override fun readOrNull(path: String): ByteBuffer? {
val node = indexNodes[path] ?: return null
return node.read()
}
override fun close() {
reader.close()
}
}