From abe63ae1ffd00c61369b0d83ff6c88dc534482d2 Mon Sep 17 00:00:00 2001 From: Gleb Mazovetskiy Date: Mon, 10 Jul 2023 05:10:50 +0100 Subject: [PATCH] Re-encode CL2 Original Blizzard encoder is slightly less optimal than our encoder. Savings for unpacked and minified MPQs: * diabdat.mpq: 918,311 bytes. * hellfire.mpq: 313,882 bytes. Example player graphics (note that only a few are loaded at any given time for single player): * diabdat/plrgfx/warrior/: 366,564 bytes. Example monster graphics: * diabdat/monsters/skelbow: 5,391 bytes. Fixes #5 --- CMakeLists.txt | 2 +- src/internal/cel2clx.cpp | 12 +-- src/internal/cl22clx.cpp | 184 +++++++++++++++++++++++++++++---- src/internal/cl22clx_main.cpp | 11 +- src/public/include/cl22clx.hpp | 30 ++++-- 5 files changed, 204 insertions(+), 35 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index bc03a1a..0a34055 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -100,7 +100,7 @@ add_library( ) add_library(DvlGfx::cl22clx ALIAS cl22clx) target_link_libraries(cl22clx PUBLIC common) -target_link_libraries(cel2clx PRIVATE clx_encode) +target_link_libraries(cl22clx PRIVATE clx_encode) set_target_properties(cl22clx PROPERTIES PUBLIC_HEADER "src/public/include/cl22clx.hpp") target_include_directories(cl22clx PRIVATE src/internal) diff --git a/src/internal/cel2clx.cpp b/src/internal/cel2clx.cpp index 4a55adc..58d51ba 100644 --- a/src/internal/cel2clx.cpp +++ b/src/internal/cel2clx.cpp @@ -62,16 +62,16 @@ std::optional CelToClx(const uint8_t *data, size_t size, WriteLE32(&clxData[4 * group], clxData.size()); } - // CL2 header: frame count, frame offset for each frame, file size - const size_t cl2DataOffset = clxData.size(); + // CLX header: frame count, frame offset for each frame, file size + const size_t clxDataOffset = clxData.size(); clxData.resize(clxData.size() + 4 * (2 + static_cast(numFrames))); - WriteLE32(&clxData[cl2DataOffset], numFrames); + WriteLE32(&clxData[clxDataOffset], numFrames); const uint8_t *srcEnd = &data[LoadLE32(&data[4])]; for (size_t frame = 1; frame <= numFrames; ++frame) { const uint8_t *src = srcEnd; srcEnd = &data[LoadLE32(&data[4 * (frame + 1)])]; - WriteLE32(&clxData[cl2DataOffset + 4 * frame], static_cast(clxData.size() - cl2DataOffset)); + WriteLE32(&clxData[clxDataOffset + 4 * frame], static_cast(clxData.size() - clxDataOffset)); // Skip CEL frame header if there is one. constexpr size_t CelFrameHeaderSize = 10; @@ -107,12 +107,12 @@ std::optional CelToClx(const uint8_t *data, size_t size, } ++frameHeight; } + AppendClxTransparentRun(transparentRunWidth, clxData); WriteLE16(&clxData[frameHeaderPos + 4], frameHeight); memset(&clxData[frameHeaderPos + 6], 0, 4); - AppendClxTransparentRun(transparentRunWidth, clxData); } - WriteLE32(&clxData[cl2DataOffset + 4 * (1 + static_cast(numFrames))], static_cast(clxData.size() - cl2DataOffset)); + WriteLE32(&clxData[clxDataOffset + 4 * (1 + static_cast(numFrames))], static_cast(clxData.size() - clxDataOffset)); data = srcEnd; } return std::nullopt; diff --git a/src/internal/cl22clx.cpp b/src/internal/cl22clx.cpp index 4759bab..2395070 100644 --- a/src/internal/cl22clx.cpp +++ b/src/internal/cl22clx.cpp @@ -8,6 +8,7 @@ #include #include +#include #include #include @@ -15,6 +16,8 @@ namespace dvl_gfx { namespace { +constexpr size_t FrameHeaderSize = 10; + constexpr bool IsCl2Opaque(uint8_t control) { constexpr uint8_t Cl2OpaqueMin = 0x80; @@ -59,9 +62,128 @@ size_t CountCl2FramePixels(const uint8_t *src, const uint8_t *srcEnd) return numPixels; } +struct SkipSize { + int_fast16_t wholeLines; + int_fast16_t xOffset; +}; +SkipSize GetSkipSize(int_fast16_t overrun, int_fast16_t srcWidth) +{ + SkipSize result; + result.wholeLines = overrun / srcWidth; + result.xOffset = overrun - srcWidth * result.wholeLines; + return result; +} + } // namespace -std::optional Cl2ToClx(uint8_t *data, size_t size, +std::optional Cl2ToClx(const uint8_t *data, size_t size, + const uint16_t *widths, size_t numWidths, + std::vector &clxData) +{ + uint32_t numGroups = 1; + const uint32_t maybeNumFrames = LoadLE32(data); + const uint8_t *groupBegin = data; + + // If it is a number of frames, then the last frame offset will be equal to the size of the file. + if (LoadLE32(&data[maybeNumFrames * 4 + 4]) != size) { + // maybeNumFrames is the address of the first group, right after + // the list of group offsets. + numGroups = maybeNumFrames / 4; + clxData.resize(maybeNumFrames); + } + + // Transient buffer for a contiguous run of non-transparent pixels. + std::vector pixels; + pixels.reserve(4096); + + for (size_t group = 0; group < numGroups; ++group) { + uint32_t numFrames; + if (numGroups == 1) { + numFrames = maybeNumFrames; + } else { + groupBegin = &data[LoadLE32(&data[group * 4])]; + numFrames = LoadLE32(groupBegin); + WriteLE32(&clxData[4 * group], clxData.size()); + } + + // CLX header: frame count, frame offset for each frame, file size + const size_t clxDataOffset = clxData.size(); + clxData.resize(clxData.size() + 4 * (2 + static_cast(numFrames))); + WriteLE32(&clxData[clxDataOffset], numFrames); + + const uint8_t *frameEnd = &groupBegin[LoadLE32(&groupBegin[4])]; + for (size_t frame = 1; frame <= numFrames; ++frame) { + WriteLE32(&clxData[clxDataOffset + 4 * frame], + static_cast(clxData.size() - clxDataOffset)); + + const uint8_t *frameBegin = frameEnd; + frameEnd = &groupBegin[LoadLE32(&groupBegin[4 * (frame + 1)])]; + + const uint16_t frameWidth = numWidths == 1 ? *widths : widths[frame - 1]; + + const size_t frameHeaderPos = clxData.size(); + clxData.resize(clxData.size() + FrameHeaderSize); + WriteLE16(&clxData[frameHeaderPos], FrameHeaderSize); + WriteLE16(&clxData[frameHeaderPos + 2], frameWidth); + + unsigned transparentRunWidth = 0; + int_fast16_t xOffset = 0; + size_t frameHeight = 0; + const uint8_t *src = frameBegin + FrameHeaderSize; + while (src != frameEnd) { + auto remainingWidth = static_cast(frameWidth) - xOffset; + while (remainingWidth > 0) { + const ClxBlitCommand cmd = ClxGetBlitCommand(src); + switch (cmd.type) { + case ClxBlitType::Transparent: + if (!pixels.empty()) { + AppendClxPixelsOrFillRun(pixels.data(), pixels.size(), clxData); + pixels.clear(); + } + + transparentRunWidth += cmd.length; + break; + case ClxBlitType::Fill: + case ClxBlitType::Pixels: + AppendClxTransparentRun(transparentRunWidth, clxData); + transparentRunWidth = 0; + + if (cmd.type == ClxBlitType::Fill) { + pixels.insert(pixels.end(), cmd.length, cmd.color); + } else { // ClxBlitType::Pixels + pixels.insert(pixels.end(), src + 1, cmd.srcEnd); + } + break; + } + src = cmd.srcEnd; + remainingWidth -= cmd.length; + } + + ++frameHeight; + if (remainingWidth < 0) { + const auto skipSize = GetSkipSize(-remainingWidth, static_cast(frameWidth)); + xOffset = skipSize.xOffset; + frameHeight += skipSize.wholeLines; + } else { + xOffset = 0; + } + } + if (!pixels.empty()) { + AppendClxPixelsOrFillRun(pixels.data(), pixels.size(), clxData); + pixels.clear(); + } + AppendClxTransparentRun(transparentRunWidth, clxData); + + WriteLE16(&clxData[frameHeaderPos + 4], frameHeight); + memset(&clxData[frameHeaderPos + 6], 0, 4); + } + + WriteLE32(&clxData[clxDataOffset + 4 * (1 + static_cast(numFrames))], static_cast(clxData.size() - clxDataOffset)); + } + return std::nullopt; +} + +std::optional Cl2ToClxNoReencode(uint8_t *data, size_t size, const uint16_t *widths, size_t numWidths) { uint32_t numGroups = 1; @@ -89,8 +211,7 @@ std::optional Cl2ToClx(uint8_t *data, size_t size, uint8_t *frameBegin = frameEnd; frameEnd = &groupBegin[LoadLE32(&groupBegin[4 * (frame + 1)])]; - constexpr size_t Cl2FrameHeaderSize = 10; - const size_t numPixels = CountCl2FramePixels(frameBegin + Cl2FrameHeaderSize, frameEnd); + const size_t numPixels = CountCl2FramePixels(frameBegin + FrameHeaderSize, frameEnd); const uint16_t frameWidth = numWidths == 1 ? *widths : widths[frame - 1]; const uint16_t frameHeight = numPixels / frameWidth; @@ -103,7 +224,7 @@ std::optional Cl2ToClx(uint8_t *data, size_t size, } std::optional Cl2ToClx(const char *inputPath, const char *outputPath, - const uint16_t *widths, size_t numWidths) + const uint16_t *widths, size_t numWidths, bool reencode) { std::error_code ec; const uintmax_t size = std::filesystem::file_size(inputPath, ec); @@ -130,11 +251,19 @@ std::optional Cl2ToClx(const char *inputPath, const char *outputPath, return IoError { std::string("Failed to open output file: ") .append(std::strerror(errno)) }; - std::optional result = Cl2ToClx(ownedData.get(), size, widths, numWidths); - if (result.has_value()) - return result; + if (reencode) { + std::vector out; + std::optional result = Cl2ToClx(ownedData.get(), size, widths, numWidths, out); + if (result.has_value()) + return result; + output.write(reinterpret_cast(out.data()), static_cast(out.size())); + } else { + std::optional result = Cl2ToClxNoReencode(ownedData.get(), size, widths, numWidths); + if (result.has_value()) + return result; + output.write(reinterpret_cast(ownedData.get()), static_cast(size)); + } - output.write(reinterpret_cast(ownedData.get()), static_cast(size)); output.close(); if (output.fail()) return IoError { std::string("Failed to write to output file: ") @@ -144,7 +273,7 @@ std::optional Cl2ToClx(const char *inputPath, const char *outputPath, std::optional CombineCl2AsClxSheet( const char *const *inputPaths, size_t numFiles, const char *outputPath, - const std::vector &widths) + const std::vector &widths, bool reencode) { size_t accumulatedSize = ClxSheetHeaderSize(numFiles); std::vector offsets; @@ -177,19 +306,34 @@ std::optional CombineCl2AsClxSheet( } input.close(); } - if (std::optional error = Cl2ToClx( - ownedData.get(), accumulatedSize, widths.data(), widths.size()); - error.has_value()) { - return error; - } std::ofstream output; - output.open(outputPath, std::ios::out | std::ios::binary); - if (output.fail()) - return IoError { std::string("Failed to open output file: ") - .append(std::strerror(errno)) }; - - output.write(reinterpret_cast(ownedData.get()), static_cast(accumulatedSize)); + if (reencode) { + std::vector out; + if (std::optional error = Cl2ToClx( + ownedData.get(), accumulatedSize, widths.data(), widths.size(), out); + error.has_value()) { + return error; + } + output.open(outputPath, std::ios::out | std::ios::binary); + if (output.fail()) { + return IoError { std::string("Failed to open output file: ") + .append(std::strerror(errno)) }; + } + output.write(reinterpret_cast(out.data()), static_cast(out.size())); + } else { + if (std::optional error = Cl2ToClxNoReencode( + ownedData.get(), accumulatedSize, widths.data(), widths.size()); + error.has_value()) { + return error; + } + output.open(outputPath, std::ios::out | std::ios::binary); + if (output.fail()) { + return IoError { std::string("Failed to open output file: ") + .append(std::strerror(errno)) }; + } + output.write(reinterpret_cast(ownedData.get()), static_cast(accumulatedSize)); + } output.close(); if (output.fail()) return IoError { std::string("Failed to write to output file: ") diff --git a/src/internal/cl22clx_main.cpp b/src/internal/cl22clx_main.cpp index 505b301..9bf6d56 100644 --- a/src/internal/cl22clx_main.cpp +++ b/src/internal/cl22clx_main.cpp @@ -27,6 +27,7 @@ Converts CL2 sprite(s) to a CLX file. the trailing digits. --width [,...] CL2 sprite frame width(s), comma-separated. --combine Combine multiple CL2 files into a single CLX sheet. + --no-reencode Do not reencode graphics data with the more optimal DevilutionX encoder. --remove Remove the input files. -q, --quiet Do not log anything. )"; @@ -38,6 +39,7 @@ struct Options { std::vector widths; bool combine = false; bool remove = false; + bool reencode = true; bool quiet = false; }; @@ -77,6 +79,8 @@ tl::expected ParseArguments(int argc, char *argv[]) options.widths = *std::move(value); } else if (arg == "--combine") { options.combine = true; + } else if (arg == "--no-reencode") { + options.reencode = false; } else if (arg == "--remove") { options.remove = true; } else if (arg == "-q" || arg == "--quiet") { @@ -140,7 +144,8 @@ std::optional Run(const Options &options) outputPath = std::filesystem::path(options.inputPaths[0]).parent_path() / outputFilename; } std::optional error = CombineCl2AsClxSheet( - options.inputPaths.data(), options.inputPaths.size(), outputPath.string().c_str(), options.widths); + options.inputPaths.data(), options.inputPaths.size(), + outputPath.string().c_str(), options.widths, options.reencode); if (error.has_value()) return error; return std::nullopt; @@ -159,7 +164,9 @@ std::optional Run(const Options &options) } else { outputPath = inputPathFs.parent_path() / outputFilename; } - if (std::optional error = Cl2ToClx(inputPath, outputPath.string().c_str(), options.widths); + if (std::optional error = Cl2ToClx( + inputPath, outputPath.string().c_str(), + options.widths, options.reencode); error.has_value()) { error->message.append(": ").append(inputPath); return error; diff --git a/src/public/include/cl22clx.hpp b/src/public/include/cl22clx.hpp index 658d746..fa4d8a9 100644 --- a/src/public/include/cl22clx.hpp +++ b/src/public/include/cl22clx.hpp @@ -12,7 +12,24 @@ namespace dvl_gfx { /** - * @brief Converts a CL2 image to CLX in-place. + * @brief Converts a CL2 image to CLX. + * + * Re-encodes the frames. This can reduce file size because the dvl_gfx encoder + * is more optimal than the original encoder used by Blizzard. + * + * @param data The CL2 buffer. + * @param size CL2 buffer size. + * @param widths Widths of each frame. If all the frame are the same width, this can be a single number. + * @param numWidths The number of widths. + * @return std::optional + */ +std::optional Cl2ToClx(const uint8_t *data, size_t size, + const uint16_t *widths, size_t numWidths, std::vector &out); + +/** + * @brief Converts a CL2 image to CLX in-place without re-encoding. + * + * Does not re-encode the frames. * * @param data The CL2 buffer. * @param size CL2 buffer size. @@ -20,16 +37,16 @@ namespace dvl_gfx { * @param numWidths The number of widths. * @return std::optional */ -std::optional Cl2ToClx(uint8_t *data, size_t size, +std::optional Cl2ToClxNoReencode(uint8_t *data, size_t size, const uint16_t *widths, size_t numWidths); std::optional Cl2ToClx(const char *inputPath, const char *outputPath, - const uint16_t *widths, size_t numWidths); + const uint16_t *widths, size_t numWidths, bool reencode = true); inline std::optional Cl2ToClx(const char *inputPath, const char *outputPath, - const std::vector &widths) + const std::vector &widths, bool reencode = true) { - return Cl2ToClx(inputPath, outputPath, widths.data(), widths.size()); + return Cl2ToClx(inputPath, outputPath, widths.data(), widths.size(), reencode); } /** @@ -38,11 +55,12 @@ inline std::optional Cl2ToClx(const char *inputPath, const char *output * @param inputPaths Paths to the input files. * @param numFiles The number of `inputPaths`. * @param widths Widths of each frame. If all the frame are the same width, this can be a single number. + * @param reencode If true, reencodes the CL2 graphics data (our encoder produces slightly smaller files). * @return std::optional */ std::optional CombineCl2AsClxSheet( const char *const *inputPaths, size_t numFiles, const char *outputPath, - const std::vector &widths); + const std::vector &widths, bool reencode = true); } // namespace dvl_gfx #endif // DVL_GFX_CL22CLX_H_