Very compact dungeon representation in RAM, but it is quite slow

src/main/java/ru/dbotthepony/kstarbound/defs/dungeon/VectorizedBitSet.java

@@ -0,0 +1,245 @@
+package ru.dbotthepony.kstarbound.defs.dungeon;
+
+import java.util.BitSet;
+
+// TODO: actually make it vectorized, some unsafe API maybe?
+// currently it is quite slow, but achieves desired memory efficiency.
+public final class VectorizedBitSet {
+	public final int bits;
+	public final int width;
+	public final int height;
+	private final BitSet data;
+
+	public VectorizedBitSet(int bits, int width, int height) {
+		if (bits <= 0 || bits >= 13)
+			throw new IllegalArgumentException("Too many or no bits: " + bits + ". Maximum 12 supported");
+
+		this.bits = bits;
+		this.width = width;
+		this.height = height;
+
+		this.data = new BitSet(bits * width * height);
+	}
+
+	private static int bit(boolean value, int order) {
+		return (value ? 1 : 0) << order;
+	}
+
+	private void bit(int index, int value, int order) {
+		if (((value >>> order) & 1) == 0) {
+			data.clear(index);
+		} else {
+			data.set(index);
+		}
+	}
+
+	public void set(int x, int y, int value) {
+		int bitIndex = (x + y * width) * bits;
+
+		switch (this.bits) {
+			case 0:
+
+			case 1:
+				bit(bitIndex, value, 0);
+				break;
+
+			case 2:
+				bit(bitIndex, value, 0);
+				bit(bitIndex + 1, value, 1);
+				break;
+
+			case 3:
+				bit(bitIndex, value, 0);
+				bit(bitIndex + 1, value, 1);
+				bit(bitIndex + 2, value, 2);
+				break;
+
+			case 4:
+				bit(bitIndex, value, 0);
+				bit(bitIndex + 1, value, 1);
+				bit(bitIndex + 2, value, 2);
+				bit(bitIndex + 3, value, 3);
+				break;
+
+			case 5:
+				bit(bitIndex, value, 0);
+				bit(bitIndex + 1, value, 1);
+				bit(bitIndex + 2, value, 2);
+				bit(bitIndex + 3, value, 3);
+				bit(bitIndex + 4, value, 4);
+				break;
+
+			case 6:
+				bit(bitIndex, value, 0);
+				bit(bitIndex + 1, value, 1);
+				bit(bitIndex + 2, value, 2);
+				bit(bitIndex + 3, value, 3);
+				bit(bitIndex + 4, value, 4);
+				bit(bitIndex + 5, value, 5);
+				break;
+
+			case 7:
+				bit(bitIndex, value, 0);
+				bit(bitIndex + 1, value, 1);
+				bit(bitIndex + 2, value, 2);
+				bit(bitIndex + 3, value, 3);
+				bit(bitIndex + 4, value, 4);
+				bit(bitIndex + 5, value, 5);
+				bit(bitIndex + 6, value, 6);
+				break;
+
+			case 8:
+				bit(bitIndex, value, 0);
+				bit(bitIndex + 1, value, 1);
+				bit(bitIndex + 2, value, 2);
+				bit(bitIndex + 3, value, 3);
+				bit(bitIndex + 4, value, 4);
+				bit(bitIndex + 5, value, 5);
+				bit(bitIndex + 6, value, 6);
+				bit(bitIndex + 7, value, 7);
+				break;
+
+			case 9:
+				bit(bitIndex, value, 0);
+				bit(bitIndex + 1, value, 1);
+				bit(bitIndex + 2, value, 2);
+				bit(bitIndex + 3, value, 3);
+				bit(bitIndex + 4, value, 4);
+				bit(bitIndex + 5, value, 5);
+				bit(bitIndex + 6, value, 6);
+				bit(bitIndex + 7, value, 7);
+				bit(bitIndex + 8, value, 8);
+				break;
+
+			case 10:
+				bit(bitIndex, value, 0);
+				bit(bitIndex + 1, value, 1);
+				bit(bitIndex + 2, value, 2);
+				bit(bitIndex + 3, value, 3);
+				bit(bitIndex + 4, value, 4);
+				bit(bitIndex + 5, value, 5);
+				bit(bitIndex + 6, value, 6);
+				bit(bitIndex + 7, value, 7);
+				bit(bitIndex + 8, value, 8);
+				bit(bitIndex + 9, value, 9);
+				break;
+
+			case 11:
+				bit(bitIndex, value, 0);
+				bit(bitIndex + 1, value, 1);
+				bit(bitIndex + 2, value, 2);
+				bit(bitIndex + 3, value, 3);
+				bit(bitIndex + 4, value, 4);
+				bit(bitIndex + 5, value, 5);
+				bit(bitIndex + 6, value, 6);
+				bit(bitIndex + 7, value, 7);
+				bit(bitIndex + 8, value, 8);
+				bit(bitIndex + 9, value, 9);
+				bit(bitIndex + 10, value, 10);
+				break;
+
+			case 12:
+				bit(bitIndex, value, 0);
+				bit(bitIndex + 1, value, 1);
+				bit(bitIndex + 2, value, 2);
+				bit(bitIndex + 3, value, 3);
+				bit(bitIndex + 4, value, 4);
+				bit(bitIndex + 5, value, 5);
+				bit(bitIndex + 6, value, 6);
+				bit(bitIndex + 7, value, 7);
+				bit(bitIndex + 8, value, 8);
+				bit(bitIndex + 9, value, 9);
+				bit(bitIndex + 10, value, 10);
+				bit(bitIndex + 11, value, 11);
+				break;
+
+		}
+	}
+
+	public int get(int x, int y) {
+		int bitIndex = (x + y * width) * bits;
+		BitSet data = this.data;
+
+		return switch (this.bits) {
+			case 0, 1 -> bit(data.get(bitIndex), 0);
+			case 2 -> bit(data.get(bitIndex), 0) |
+				bit(data.get(bitIndex + 1), 1);
+			case 3 -> bit(data.get(bitIndex), 0) |
+				bit(data.get(bitIndex + 1), 1) |
+				bit(data.get(bitIndex + 2), 2);
+			case 4 -> bit(data.get(bitIndex), 0) |
+				bit(data.get(bitIndex + 1), 1) |
+				bit(data.get(bitIndex + 2), 2) |
+				bit(data.get(bitIndex + 3), 3);
+			case 5 -> bit(data.get(bitIndex), 0) |
+				bit(data.get(bitIndex + 1), 1) |
+				bit(data.get(bitIndex + 2), 2) |
+				bit(data.get(bitIndex + 3), 3) |
+				bit(data.get(bitIndex + 4), 4);
+			case 6 -> bit(data.get(bitIndex), 0) |
+				bit(data.get(bitIndex + 1), 1) |
+				bit(data.get(bitIndex + 2), 2) |
+				bit(data.get(bitIndex + 3), 3) |
+				bit(data.get(bitIndex + 4), 4) |
+				bit(data.get(bitIndex + 5), 5);
+			case 7 -> bit(data.get(bitIndex), 0) |
+				bit(data.get(bitIndex + 1), 1) |
+				bit(data.get(bitIndex + 2), 2) |
+				bit(data.get(bitIndex + 3), 3) |
+				bit(data.get(bitIndex + 4), 4) |
+				bit(data.get(bitIndex + 5), 5) |
+				bit(data.get(bitIndex + 6), 6);
+			case 8 -> bit(data.get(bitIndex), 0) |
+				bit(data.get(bitIndex + 1), 1) |
+				bit(data.get(bitIndex + 2), 2) |
+				bit(data.get(bitIndex + 3), 3) |
+				bit(data.get(bitIndex + 4), 4) |
+				bit(data.get(bitIndex + 5), 5) |
+				bit(data.get(bitIndex + 6), 6) |
+				bit(data.get(bitIndex + 7), 7);
+			case 9 -> bit(data.get(bitIndex), 0) |
+				bit(data.get(bitIndex + 1), 1) |
+				bit(data.get(bitIndex + 2), 2) |
+				bit(data.get(bitIndex + 3), 3) |
+				bit(data.get(bitIndex + 4), 4) |
+				bit(data.get(bitIndex + 5), 5) |
+				bit(data.get(bitIndex + 6), 6) |
+				bit(data.get(bitIndex + 7), 7) |
+				bit(data.get(bitIndex + 8), 8);
+			case 10 -> bit(data.get(bitIndex), 0) |
+				bit(data.get(bitIndex + 1), 1) |
+				bit(data.get(bitIndex + 2), 2) |
+				bit(data.get(bitIndex + 3), 3) |
+				bit(data.get(bitIndex + 4), 4) |
+				bit(data.get(bitIndex + 5), 5) |
+				bit(data.get(bitIndex + 6), 6) |
+				bit(data.get(bitIndex + 7), 7) |
+				bit(data.get(bitIndex + 8), 8) |
+				bit(data.get(bitIndex + 9), 9);
+			case 11 -> bit(data.get(bitIndex), 0) |
+				bit(data.get(bitIndex + 1), 1) |
+				bit(data.get(bitIndex + 2), 2) |
+				bit(data.get(bitIndex + 3), 3) |
+				bit(data.get(bitIndex + 4), 4) |
+				bit(data.get(bitIndex + 5), 5) |
+				bit(data.get(bitIndex + 6), 6) |
+				bit(data.get(bitIndex + 7), 7) |
+				bit(data.get(bitIndex + 8), 8) |
+				bit(data.get(bitIndex + 9), 9) |
+				bit(data.get(bitIndex + 10), 10);
+			case 12 -> bit(data.get(bitIndex), 0) |
+				bit(data.get(bitIndex + 1), 1) |
+				bit(data.get(bitIndex + 2), 2) |
+				bit(data.get(bitIndex + 3), 3) |
+				bit(data.get(bitIndex + 4), 4) |
+				bit(data.get(bitIndex + 5), 5) |
+				bit(data.get(bitIndex + 6), 6) |
+				bit(data.get(bitIndex + 7), 7) |
+				bit(data.get(bitIndex + 8), 8) |
+				bit(data.get(bitIndex + 9), 9) |
+				bit(data.get(bitIndex + 10), 10) |
+				bit(data.get(bitIndex + 11), 11);
+			default -> 0;
+		};
+	}
+}

src/main/kotlin/ru/dbotthepony/kstarbound/defs/dungeon/ImagePartReader.kt

@@ -1,51 +1,101 @@
 package ru.dbotthepony.kstarbound.defs.dungeon
 
 import com.google.common.collect.ImmutableList
+import it.unimi.dsi.fastutil.ints.IntAVLTreeSet
+import it.unimi.dsi.fastutil.ints.IntArrayList
+import it.unimi.dsi.fastutil.objects.ObjectArrayList
+import org.lwjgl.stb.STBImage
+import org.lwjgl.system.MemoryUtil
 import ru.dbotthepony.kommons.arrays.Object2DArray
 import ru.dbotthepony.kommons.math.RGBAColor
 import ru.dbotthepony.kommons.util.KOptional
 import ru.dbotthepony.kommons.vector.Vector2i
+import ru.dbotthepony.kstarbound.Starbound
 import ru.dbotthepony.kstarbound.defs.image.Image
+import java.lang.ref.Reference
 
 class ImagePartReader(part: DungeonPart, val images: ImmutableList<Image>) : PartReader(part) {
 	override val size: Vector2i
 		get() = if (images.isEmpty()) Vector2i.ZERO else images.first().size
 
-	// it is much cheaper to just read all images and store 2D array
+	// ObjectArrayList doesn't check for concurrent modifications
-	// of references than loading / keeping images themselves around
+	private val layers = ObjectArrayList<Layer>()
-	// `Image` class doesn't actually keep pixel data around for too long,
+	private class Layer(val palette: Array<DungeonTile>, val data: VectorizedBitSet)
-	// if it doesn't get accessed in some time it gets purged from ram
-	private val layers = Array(images.size) {
-		Object2DArray.nulls<DungeonTile>(images[it].width, images[it].height)
-	} as Array<Object2DArray<DungeonTile>>
 
 	override fun bind(def: DungeonDefinition) {
 		check(def.tiles.isNotEmpty) { "Image parts require 'tiles' palette to be present in .dungeon definition" }
 
 		for ((i, image) in images.withIndex()) {
-			val layer = layers[i]
+			// go around image cache, since image will be loaded exactly once
+			// and then forgotten
+			val (bytes, width, height, channels) = Image.readImageDirect(image.source)
+			val tileData = IntArray(width * height)
+
+			if (channels == 3) {
+				// RGB
+				for (x in 0 until image.width) {
+					for (y in 0 until image.height) {
+						val offset = (x + y * image.width) * channels
+
+						tileData[x + y * image.width] = bytes[offset].toInt().and(0xFF) or
+							bytes[offset + 1].toInt().and(0xFF).shl(8) or
+							bytes[offset + 2].toInt().and(0xFF).shl(16) or -0x1000000 // leading alpha as 255
+					}
+				}
+			} else if (channels == 4) {
+				// RGBA
+
+				for (x in 0 until image.width) {
+					for (y in 0 until image.height) {
+						val offset = (x + y * image.width) * channels
+
+						tileData[x + y * image.width] = bytes[offset].toInt().and(0xFF) or
+							bytes[offset + 1].toInt().and(0xFF).shl(8) or
+							bytes[offset + 2].toInt().and(0xFF).shl(16) or
+							bytes[offset + 3].toInt().and(0xFF).shl(24)
+					}
+				}
+			}
+
+			// determine unique tiles
+			val uniqueTiles = IntAVLTreeSet()
 
 			for (y in 0 until image.height) {
 				for (x in 0 until image.width) {
-					val color = image[x, y]
+					val color = tileData[x + y * image.width]
-					val tile = part.dungeon.tiles[color]
 
-					if (tile == null) {
+					if (uniqueTiles.add(color)) {
-						val parse = RGBAColor.abgr(color)
+						if (part.dungeon.tiles[color] == null) {
-						throw IllegalStateException("Unknown tile on ${image.path} at $x, $y: [${parse.redInt}, ${parse.greenInt}, ${parse.blueInt}, ${parse.alphaInt}] (index $color)")
+							val parse = RGBAColor.abgr(color)
+							throw IllegalStateException("Unknown tile inside ${image.path} at $x, $y with color [${parse.redInt}, ${parse.greenInt}, ${parse.blueInt}, ${parse.alphaInt}] (index $color)")
+						}
 					}
-
-					layer[x, y] = tile
 				}
 			}
+
+			// construct palette
+			val indices = IntArrayList(uniqueTiles)
+			val palette = Array(indices.size) { part.dungeon.tiles[indices.getInt(it)]!! }
+			val data = VectorizedBitSet(32 - Integer.numberOfLeadingZeros(uniqueTiles.size), image.width, image.height)
+
+			// fill data
+			for (y in 0 until image.height) {
+				for (x in 0 until image.width) {
+					val index = indices.indexOf(tileData[x + y * image.width])
+					check(index != -1)
+					data[x, y] = index
+				}
+			}
+
+			layers.add(Layer(palette, data))
 		}
 	}
 
 	override fun <T> walkTiles(callback: TileCallback<T>): KOptional<T> {
 		for (layer in layers) {
-			for (y in 0 until layer.rows) {
+			for (y in 0 until layer.data.height) {
-				for (x in 0 until layer.columns) {
+				for (x in 0 until layer.data.width) {
-					val get = callback(x, y, layer[x, y])
+					val get = callback(x, y, layer.palette[layer.data[x, y]])
 					if (get.isPresent) return get
 				}
 			}
@@ -56,8 +106,8 @@ class ImagePartReader(part: DungeonPart, val images: ImmutableList<Image>) : Par
 
 	override fun <T> walkTilesAt(x: Int, y: Int, callback: TileCallback<T>): KOptional<T> {
 		for (layer in layers) {
-			if (x in 0 until layer.columns && y in 0 until layer.rows) {
+			if (x in 0 until layer.data.width && y in 0 until layer.data.height) {
-				val get = callback(x, y, layer[x, y])
+				val get = callback(x, y, layer.palette[layer.data[x, y]])
 				if (get.isPresent) return get
 			}
 		}

src/main/kotlin/ru/dbotthepony/kstarbound/defs/dungeon/TiledMap.kt

@@ -5,11 +5,12 @@ import com.google.gson.JsonElement
 import com.google.gson.JsonNull
 import com.google.gson.JsonObject
 import com.google.gson.JsonPrimitive
+import it.unimi.dsi.fastutil.ints.Int2IntOpenHashMap
+import it.unimi.dsi.fastutil.ints.IntAVLTreeSet
+import it.unimi.dsi.fastutil.ints.IntArrayList
 import it.unimi.dsi.fastutil.io.FastByteArrayInputStream
 import ru.dbotthepony.kommons.gson.contains
-import ru.dbotthepony.kommons.gson.get
 import ru.dbotthepony.kommons.gson.set
-import ru.dbotthepony.kommons.util.AABBi
 import ru.dbotthepony.kommons.util.Either
 import ru.dbotthepony.kommons.util.KOptional
 import ru.dbotthepony.kommons.vector.Vector2i
@@ -48,7 +49,7 @@ class TiledMap(data: JsonData) : TileMap() {
 
 	val size = Vector2i(data.width, data.height)
 
-	val tileSets = TiledTileSets(data.tilesets)
+	private val tileSets = TiledTileSets(data.tilesets)
 
 	var frontLayer: TileLayer? = null
 		private set
@@ -93,6 +94,8 @@ class TiledMap(data: JsonData) : TileMap() {
 		}
 
 		this.objectLayers = ImmutableList.copyOf(objectLayers)
+
+		tileSets.free()
 	}
 
 	override fun <T> walkTiles(callback: TileCallback<T>): KOptional<T> {
@@ -145,10 +148,13 @@ class TiledMap(data: JsonData) : TileMap() {
 		val x: Int = data.x
 		val y: Int = data.y
 
-		// this eats ram, need to use cache or huffman encoding with bitset
+		private val frontPalette: Array<DungeonTile>
-		private val tileData: IntArray
+		private val backPalette: Array<DungeonTile>
+		private val data: VectorizedBitSet
 
 		init {
+			val tileData: IntArray
+
 			if (data.compression == "zlib") {
 				val stream = BufferedInputStream(InflaterInputStream(FastByteArrayInputStream(Base64.getDecoder().decode(data.data.asString))))
 
@@ -176,17 +182,39 @@ class TiledMap(data: JsonData) : TileMap() {
 			} else {
 				throw IllegalArgumentException("Unsupported compression mode: ${data.compression}")
 			}
+
+			// determine unique tiles
+			val countUniqueTiles = IntAVLTreeSet()
+
+			for (x in 0 until width) {
+				for (y in 0 until height) {
+					countUniqueTiles.add(tileData[x + y * width])
+				}
+			}
+
+			// construct backing memory
+			this.data = VectorizedBitSet(32 - Integer.numberOfLeadingZeros(countUniqueTiles.size), width, height)
+
+			// determine palette
+			val gidIndices = IntArrayList(countUniqueTiles)
+			frontPalette = Array(gidIndices.size) { tileSets.getFront(gidIndices.getInt(it)) }
+			backPalette = Array(gidIndices.size) { tileSets.getBack(gidIndices.getInt(it)) }
+
+			// fill backing memory with paletted indices
+			for (x in 0 until width) {
+				for (y in 0 until height) {
+					val index = gidIndices.indexOf(tileData[x + y * width])
+					check(index != -1)
+					this.data[x, this.height - y - 1] = index
+				}
+			}
 		}
 
 		private fun get0(x: Int, y: Int): DungeonTile {
-			val actualX = x - this.x
-			var actualY = y - this.y
-			actualY = this.height - actualY - 1
-
 			if (isBackground)
-				return tileSets.getBack(tileData[actualX + actualY * width])
+				return backPalette[data[x - this.x, y - this.y]]
 			else
-				return tileSets.getFront(tileData[actualX + actualY * width])
+				return frontPalette[data[x - this.x, y - this.y]]
 		}
 
 		operator fun get(x: Int, y: Int): DungeonTile {

src/main/kotlin/ru/dbotthepony/kstarbound/defs/dungeon/TiledTileSets.kt

@@ -14,8 +14,8 @@ class TiledTileSets(entries: List<Entry>) {
 		val source: String,
 	)
 
-	private val front = Int2ObjectOpenHashMap<Pair<DungeonTile, JsonObject>>()
+	private var front = Int2ObjectOpenHashMap<Pair<DungeonTile, JsonObject>>()
-	private val back = Int2ObjectOpenHashMap<Pair<DungeonTile, JsonObject>>()
+	private var back = Int2ObjectOpenHashMap<Pair<DungeonTile, JsonObject>>()
 
 	init {
 		for ((firstgid, source) in entries) {
@@ -60,4 +60,9 @@ class TiledTileSets(entries: List<Entry>) {
 	fun getBackData(gid: Int): JsonObject {
 		return back[gid]?.second ?: JsonObject()
 	}
+
+	fun free() {
+		front = Int2ObjectOpenHashMap()
+		back = Int2ObjectOpenHashMap()
+	}
 }

src/main/kotlin/ru/dbotthepony/kstarbound/defs/image/Image.kt

@@ -341,6 +341,29 @@ class Image private constructor(
 		private val imageCache = ConcurrentHashMap<String, Optional<Image>>()
 		private val logger = LogManager.getLogger()
 
+		data class ReadDirectData(val data: ByteBuffer, val width: Int, val height: Int, val channels: Int)
+
+		fun readImageDirect(file: IStarboundFile): ReadDirectData {
+			val getWidth = intArrayOf(0)
+			val getHeight = intArrayOf(0)
+			val components = intArrayOf(0)
+
+			val idata = file.readDirect()
+
+			val data = STBImage.stbi_load_from_memory(
+				idata,
+				getWidth, getHeight,
+				components, 0
+			) ?: throw IllegalArgumentException("File $file is not an image or it is corrupted")
+
+			Reference.reachabilityFence(idata)
+
+			val address = MemoryUtil.memAddress(data)
+			Starbound.CLEANER.register(data) { STBImage.nstbi_image_free(address) }
+
+			return ReadDirectData(data, getWidth[0], getHeight[0], components[0])
+		}
+
 		private val dataCache: AsyncLoadingCache<IStarboundFile, ByteBuffer> = Caffeine.newBuilder()
 			.expireAfterAccess(Duration.ofMinutes(1))
 			.weigher<IStarboundFile, ByteBuffer> { key, value -> value.capacity() }
@@ -348,24 +371,7 @@ class Image private constructor(
 			.scheduler(Starbound)
 			.executor(Starbound.EXECUTOR)
 			.buildAsync(CacheLoader {
-				val getWidth = intArrayOf(0)
+				readImageDirect(it).data
-				val getHeight = intArrayOf(0)
-				val components = intArrayOf(0)
-
-				val idata = it.readDirect()
-
-				val data = STBImage.stbi_load_from_memory(
-					idata,
-					getWidth, getHeight,
-					components, 0
-				) ?: throw IllegalArgumentException("File $it is not an image or it is corrupted")
-
-				Reference.reachabilityFence(idata)
-
-				val address = MemoryUtil.memAddress(data)
-				Starbound.CLEANER.register(data) { STBImage.nstbi_image_free(address) }
-
-				data
 			})
 
 		@JvmStatic