Merge branch 'main' into vtfpp-endian

Author: partyvan

CMakeLists.txt

@@ -251,7 +251,8 @@ if(SOURCEPP_BUILD_PYTHON_WRAPPERS)
             "sourcepp._sourcepp_impl.vtfpp"
             "sourcepp._sourcepp_impl.vtfpp.ImageFormatDetails"
             "sourcepp._sourcepp_impl.vtfpp.ImageDimensions"
-            "sourcepp._sourcepp_impl.vtfpp.ImageConversion")
+            "sourcepp._sourcepp_impl.vtfpp.ImageConversion"
+            "sourcepp._sourcepp_impl.vtfpp.ImageQuantize")
     foreach(MODULE ${${SOURCEPP_PYTHON_NAME}_MODULES})
         string(REPLACE "." "/" MODULE_DIR "${MODULE}")
         string(REPLACE "." "_" MODULE_NAME_NORMALIZED "${MODULE}")

FUTURE.md

@@ -10,7 +10,7 @@
     - There is literally nothing else that supports BC6H
 - `bsppp`
   - Console paklump support
-    - Note that XBOX uses PC format. X360, PS3 do not
+    - Note that XBOX uses XZP lump, X360, PS3 use PC lump but console style zip
   - Support parsing more lumps
   - Think about GoldSrc support
 - `dmxpp`
@@ -42,6 +42,3 @@
   - XZP write support
 - `vtfpp`
   - Add a MKS parser/writer
-  - Add DPID resize filter
-  - Allow directly applying an HDRI to a VTF without requiring the user to write a custom wrapper
-  - XTF support

README.md

@@ -393,6 +393,14 @@ Several modern C++20 libraries for sanely parsing Valve formats, rolled into one
     <td align="center">✅</td>
     <td align="center">✅</td>
   </tr>
+  <tr><!-- empty row to disable github striped bg color --></tr>
+  <tr>
+    <td>
+      <a href="https://developer.valvesoftware.com/wiki/Half-Life_2_(Xbox)/Modding_Guide">XTF</a> v5.0
+    </td>
+    <td align="center">✅</td>
+    <td align="center">✅</td>
+  </tr>
 </table>
 
 (\*) These libraries are incomplete and still in development. Their interfaces are unstable and will likely change in the future.

docs/index.md

@@ -342,6 +342,13 @@ Several modern C++20 libraries for sanely parsing Valve formats, rolled into one
     <td align="center">✅</td>
     <td align="center">✅</td>
   </tr>
+  <tr>
+    <td>
+      <a href="https://developer.valvesoftware.com/wiki/Half-Life_2_(Xbox)/Modding_Guide">XTF</a> v5.0
+    </td>
+    <td align="center">✅</td>
+    <td align="center">✅</td>
+  </tr>
 </table>
 \endhtmlonly
 

ext/bufferstream

@@ -734,6 +734,20 @@ namespace ImageFormatDetails {
 	return length;
 }
 
+// XTF (PLATFORM_XBOX) has padding between frames to align each one to 512 bytes
+[[nodiscard]] constexpr uint32_t getDataLengthXBOX(bool padded, ImageFormat format, uint8_t mipCount, uint16_t frameCount, uint8_t faceCount, uint16_t width, uint16_t height, uint16_t sliceCount = 1) {
+	uint32_t length = 0;
+	for (int j = 0; j < frameCount; j++) {
+		for (int i = 0; i < mipCount; i++) {
+			length += ImageFormatDetails::getDataLength(format, ImageDimensions::getMipDim(i, width), ImageDimensions::getMipDim(i, height), sliceCount) * faceCount;
+		}
+		if (padded && j + 1 != frameCount && length > 512) {
+			length += sourcepp::math::paddingForAlignment(512, length);
+		}
+	}
+	return length;
+}
+
 [[nodiscard]] constexpr bool getDataPosition(uint32_t& offset, uint32_t& length, ImageFormat format, uint8_t mip, uint8_t mipCount, uint16_t frame, uint16_t frameCount, uint8_t face, uint8_t faceCount, uint16_t width, uint16_t height, uint16_t slice = 0, uint16_t sliceCount = 1) {
 	offset = 0;
 	length = 0;
@@ -745,16 +759,39 @@ namespace ImageFormatDetails {
 					if (i == mip && j == frame && k == face && l == slice) {
 						length = imageSize;
 						return true;
-					} else {
-						offset += imageSize;
 					}
+					offset += imageSize;
 				}
 			}
 		}
 	}
 	return false;
 }
 
+// XTF (PLATFORM_XBOX) is more like DDS layout
+[[nodiscard]] constexpr bool getDataPositionXbox(uint32_t& offset, uint32_t& length, bool padded, ImageFormat format, uint8_t mip, uint8_t mipCount, uint16_t frame, uint16_t frameCount, uint8_t face, uint8_t faceCount, uint16_t width, uint16_t height, uint16_t slice = 0, uint16_t sliceCount = 1) {
+	offset = 0;
+	length = 0;
+	for (int j = 0; j < frameCount; j++) {
+		for (int k = 0; k < faceCount; k++) {
+			for (int i = 0; i < mipCount; i++) {
+				for (int l = 0; l < sliceCount; l++) {
+					const auto imageSize = ImageFormatDetails::getDataLength(format, ImageDimensions::getMipDim(i, width), ImageDimensions::getMipDim(i, height));
+					if (i == mip && j == frame && k == face && l == slice) {
+						length = imageSize;
+						return true;
+					}
+					offset += imageSize;
+				}
+			}
+		}
+		if (padded && j + 1 != frameCount && offset > 512) {
+			offset += sourcepp::math::paddingForAlignment(512, offset);
+		}
+	}
+	return false;
+}
+
 } // namespace ImageFormatDetails
 
 } // namespace vtfpp

include/vtfpp/ImageFormats.h

@@ -0,0 +1,10 @@
+#pragma once
+
+#include "ImageConversion.h"
+
+namespace vtfpp::ImageQuantize {
+
+/// Converts a paletted image to something usable.
+[[nodiscard]] std::vector<std::byte> convertP8ImageDataToBGRA8888(std::span<const std::byte> paletteData, std::span<const std::byte> imageData);
+
+} // namespace vtfpp::ImageQuantize

include/vtfpp/ImageQuantize.h

@@ -20,6 +20,7 @@
 namespace vtfpp {
 
 constexpr uint32_t VTF_SIGNATURE = sourcepp::parser::binary::makeFourCC("VTF\0");
+constexpr uint32_t XTF_SIGNATURE = sourcepp::parser::binary::makeFourCC("XTF\0");
 constexpr uint32_t VTFX_SIGNATURE = sourcepp::parser::binary::makeFourCC("VTFX");
 constexpr uint32_t VTF3_SIGNATURE = sourcepp::parser::binary::makeFourCC("VTF3");
 
@@ -34,8 +35,10 @@ enum class CompressionMethod : int16_t {
 
 struct Resource {
 	enum Type : uint32_t {
-		TYPE_UNKNOWN             = 0,   // Unknown
+		TYPE_UNKNOWN             = 0, // Unknown
 		TYPE_THUMBNAIL_DATA      = 1,
+		TYPE_PALETTE_DATA        = 2, // Hack for XBOX platform
+		TYPE_FALLBACK_DATA       = 3, // Hack for XBOX platform
 		TYPE_PARTICLE_SHEET_DATA = 16,
 		TYPE_HOTSPOT_DATA        = 43,
 		TYPE_IMAGE_DATA          = 48,
@@ -45,13 +48,16 @@ struct Resource {
 		TYPE_LOD_CONTROL_INFO    = sourcepp::parser::binary::makeFourCC("LOD\0"),
 		TYPE_KEYVALUES_DATA      = sourcepp::parser::binary::makeFourCC("KVD\0"),
 	};
-	static consteval std::array<Type, 9> getOrder() {
+	static consteval std::array<Type, 11> getOrder() {
 		return {
 			TYPE_THUMBNAIL_DATA,
+			TYPE_PALETTE_DATA,
+			TYPE_FALLBACK_DATA,
 			TYPE_PARTICLE_SHEET_DATA,
 			TYPE_HOTSPOT_DATA,
 			// regular Source can't handle out of order resources, but Strata can,
-			// and it's the only branch that can read this and 7.6
+			// and it's the only branch that can read this and 7.6.
+			// Put this before the image data to fix resources being cut off when skipping mips
 			TYPE_AUX_COMPRESSION,
 			TYPE_IMAGE_DATA,
 			TYPE_EXTENDED_FLAGS,
@@ -168,6 +174,12 @@ class VTF {
 	};
 	static constexpr uint32_t FLAGS_MASK_V2 = FLAG_V2_NO_DEPTH_BUFFER | FLAG_V2_CLAMP_U;
 
+	enum FlagsXBOX : uint32_t {
+		FLAG_XBOX_CACHEABLE                = 1 << 27,
+		FLAG_XBOX_UNFILTERABLE_OK          = 1 << 28,
+	};
+	static constexpr uint32_t FLAGS_MASK_XBOX = FLAG_XBOX_CACHEABLE | FLAG_XBOX_UNFILTERABLE_OK;
+
 	enum FlagsV3 : uint32_t {
 		FLAG_V3_LOAD_ALL_MIPS              = 1 << 10,
 		FLAG_V3_VERTEX_TEXTURE             = 1 << 26,
@@ -212,6 +224,7 @@ class VTF {
 	enum Platform : uint32_t {
 		PLATFORM_UNKNOWN       = 0x000,
 		PLATFORM_PC            = 0x007,
+		PLATFORM_XBOX          = 0x005,
 		PLATFORM_X360          = 0x360,
 		PLATFORM_PS3_ORANGEBOX = 0x333,
 		PLATFORM_PS3_PORTAL2   = 0x334,
@@ -239,6 +252,7 @@ class VTF {
 		float bumpMapScale = 1.f;
 		float gammaCorrection = 1.f;
 		bool invertGreenChannel = false;
+		uint8_t xboxMipScale = 0;
 	};
 
 	/// This value is only valid when passed to VTF::create through CreationOptions
@@ -363,6 +377,12 @@ class VTF {
 
 	[[nodiscard]] uint8_t getThumbnailHeight() const;
 
+	[[nodiscard]] uint8_t getFallbackWidth() const;
+
+	[[nodiscard]] uint8_t getFallbackHeight() const;
+
+	[[nodiscard]] uint8_t getFallbackMipCount() const;
+
 	[[nodiscard]] const std::vector<Resource>& getResources() const;
 
 	[[nodiscard]] const Resource* getResource(Resource::Type type) const;
@@ -434,6 +454,8 @@ class VTF {
 
 	void setThumbnail(std::span<const std::byte> imageData_, ImageFormat format_, uint16_t width_, uint16_t height_, float quality = ImageConversion::DEFAULT_COMPRESSED_QUALITY);
 
+	bool setThumbnail(const std::string& imagePath, float quality = ImageConversion::DEFAULT_COMPRESSED_QUALITY); // NOLINT(*-use-nodiscard)
+
 	void computeThumbnail(ImageConversion::ResizeFilter filter = ImageConversion::ResizeFilter::DEFAULT, float quality = ImageConversion::DEFAULT_COMPRESSED_QUALITY);
 
 	void removeThumbnail();
@@ -442,6 +464,26 @@ class VTF {
 
 	bool saveThumbnailToFile(const std::string& imagePath, ImageConversion::FileFormat fileFormat = ImageConversion::FileFormat::DEFAULT) const; // NOLINT(*-use-nodiscard)
 
+	[[nodiscard]] bool hasFallbackData() const;
+
+	[[nodiscard]] std::span<const std::byte> getFallbackDataRaw(uint8_t mip = 0, uint16_t frame = 0, uint8_t face = 0) const;
+
+	[[nodiscard]] std::vector<std::byte> getFallbackDataAs(ImageFormat newFormat, uint8_t mip = 0, uint16_t frame = 0, uint8_t face = 0) const;
+
+	[[nodiscard]] std::vector<std::byte> getFallbackDataAsRGBA8888(uint8_t mip = 0, uint16_t frame = 0, uint8_t face = 0) const;
+
+	void computeFallback(ImageConversion::ResizeFilter filter = ImageConversion::ResizeFilter::DEFAULT);
+
+	void removeFallback();
+
+	[[nodiscard]] std::vector<std::byte> saveFallbackToFile(uint8_t mip = 0, uint16_t frame = 0, uint8_t face = 0, ImageConversion::FileFormat fileFormat = ImageConversion::FileFormat::DEFAULT) const;
+
+	bool saveFallbackToFile(const std::string& imagePath, uint8_t mip = 0, uint16_t frame = 0, uint8_t face = 0, ImageConversion::FileFormat fileFormat = ImageConversion::FileFormat::DEFAULT) const; // NOLINT(*-use-nodiscard)
+
+	[[nodiscard]] uint8_t getXBOXMipScale() const;
+
+	void setXBOXMipScale(uint8_t xboxMipScale_);
+
 	[[nodiscard]] std::vector<std::byte> bake() const;
 
 	bool bake(const std::string& vtfPath) const; // NOLINT(*-use-nodiscard)
@@ -461,37 +503,37 @@ class VTF {
 
 	std::vector<std::byte> data;
 
-	//uint32_t signature;
-	uint32_t version{};
-	//uint32_t headerSize;
+	uint32_t version = 4;
 
-	uint16_t width{};
-	uint16_t height{};
-	uint32_t flags{};
+	uint16_t width = 0;
+	uint16_t height = 0;
+	uint32_t flags = VTF::FLAG_NO_MIP | VTF::FLAG_NO_LOD;
 
 	uint16_t frameCount = 1;
-	uint16_t startFrame{};
+	uint16_t startFrame = 0;
 
-	//uint8_t _padding0[4];
-	sourcepp::math::Vec3f reflectivity{};
-	//uint8_t _padding1[4];
+	sourcepp::math::Vec3f reflectivity{0.2f, 0.2f, 0.2f};
 
-	float bumpMapScale{};
+	float bumpMapScale = 0.f;
 	ImageFormat format = ImageFormat::EMPTY;
 	uint8_t mipCount = 1;
 
 	ImageFormat thumbnailFormat = ImageFormat::EMPTY;
-	uint8_t thumbnailWidth{};
-	uint8_t thumbnailHeight{};
+	uint8_t thumbnailWidth = 0;
+	uint8_t thumbnailHeight = 0;
+
+	uint8_t fallbackWidth = 0;
+	uint8_t fallbackHeight = 0;
+	uint8_t fallbackMipCount = 0;
+
+	// Number of times to multiply the scale of each mip by 2 when rendering on XBOX
+	uint8_t xboxMipScale = 0;
 
 	// 1 for v7.1 and lower
 	uint16_t sliceCount = 1;
-	//uint8_t _padding2[3];
 
 	// Technically added in v7.3, but we use it to store image and thumbnail data in v7.2 and lower anyway
-	//uint32_t resourceCount;
 	std::vector<Resource> resources;
-	//uint8_t _padding3[4];
 
 	// These aren't in the header
 	Platform platform = PLATFORM_PC;

include/vtfpp/VTF.h

@@ -8,6 +8,7 @@
 #include "HOT.h"
 #include "ImageConversion.h"
 #include "ImageFormats.h"
+#include "ImageQuantize.h"
 #include "PPL.h"
 #include "SHT.h"
 #include "TTX.h"