From 99cf70515d9195f31ad59fac8111ba28052f61cd Mon Sep 17 00:00:00 2001 From: Argent77 <4519923+Argent77@users.noreply.github.com> Date: Sun, 27 Oct 2024 19:02:47 +0100 Subject: [PATCH] Improve handling of strings with non-ASCII characters in game resources --- src/org/infinity/datatype/TextString.java | 2 +- src/org/infinity/gui/GameProperties.java | 5 ++++ .../infinity/gui/menu/OptionsMenuItem.java | 8 +++--- src/org/infinity/resource/Profile.java | 26 +++++++++++++++++++ .../infinity/resource/bcs/BafResource.java | 4 +-- .../infinity/resource/bcs/BcsResource.java | 4 +-- .../infinity/resource/cre/CreResource.java | 10 +++---- .../infinity/resource/dlg/AbstractCode.java | 7 ++--- .../resource/text/PlainTextResource.java | 4 +-- src/org/infinity/util/Misc.java | 11 +------- src/org/infinity/util/StringTable.java | 2 +- 11 files changed, 53 insertions(+), 30 deletions(-) diff --git a/src/org/infinity/datatype/TextString.java b/src/org/infinity/datatype/TextString.java index 3db791e3f..9a9f417ce 100644 --- a/src/org/infinity/datatype/TextString.java +++ b/src/org/infinity/datatype/TextString.java @@ -66,7 +66,7 @@ public boolean update(Object value) { @Override public void write(OutputStream os) throws IOException { if (text != null) { - byte[] buf = text.getBytes(Misc.CHARSET_DEFAULT); + byte[] buf = text.getBytes(charset); buffer.position(0); buffer.put(buf, 0, Math.min(buf.length, buffer.limit())); while (buffer.remaining() > 0) { diff --git a/src/org/infinity/gui/GameProperties.java b/src/org/infinity/gui/GameProperties.java index 3ff1849ee..5623fb62a 100644 --- a/src/org/infinity/gui/GameProperties.java +++ b/src/org/infinity/gui/GameProperties.java @@ -227,6 +227,11 @@ private void init() { listControls.add(Couple.with(l, tf)); } + // Entry: Default charset + l = new JLabel("Default character set:"); + tf = createReadOnlyField(Profile.getDefaultCharset().displayName(), true); + listControls.add(Couple.with(l, tf)); + // Entry: Use female TLK file l = new JLabel("Uses female TLK file:"); tf = createReadOnlyField(Boolean.toString((Profile.getProperty(Profile.Key.GET_GAME_DIALOGF_FILE) != null)), true); diff --git a/src/org/infinity/gui/menu/OptionsMenuItem.java b/src/org/infinity/gui/menu/OptionsMenuItem.java index 559530130..d591a75be 100644 --- a/src/org/infinity/gui/menu/OptionsMenuItem.java +++ b/src/org/infinity/gui/menu/OptionsMenuItem.java @@ -410,9 +410,9 @@ public boolean showPreferencesDialog(Window owner) { } /** Attempts to determine the correct charset for the current game. */ - public String charsetName(String charset, boolean detect) { + public String charsetName(String charset) { if (DEFAULT_CHARSET.equalsIgnoreCase(charset)) { - charset = CharsetDetector.guessCharset(detect); + charset = Profile.getDefaultCharset().name(); } else { charset = CharsetDetector.setCharset(charset); } @@ -856,7 +856,7 @@ public boolean isCharsetAvailable(String charset) { /** Returns the character encoding of the string table. */ public String getSelectedCharset() { - return charsetName(AppOption.TLK_CHARSET_TYPE.getStringValue(), true); + return charsetName(AppOption.TLK_CHARSET_TYPE.getStringValue()); } /** Returns the currently selected game language. Returns empty string on autodetect. */ @@ -941,7 +941,7 @@ private void applyChanges(Collection options) { final String csName = option.getStringValue(); if (csName != null) { CharsetDetector.clearCache(); - StringTable.setCharset(charsetName(csName, true)); + StringTable.setCharset(charsetName(csName)); messages.add("TLK Character Encoding: " + csName); // enforce re-reading strings refresh = true; diff --git a/src/org/infinity/resource/Profile.java b/src/org/infinity/resource/Profile.java index 20348e325..ed3c872a0 100644 --- a/src/org/infinity/resource/Profile.java +++ b/src/org/infinity/resource/Profile.java @@ -11,6 +11,7 @@ import java.io.File; import java.io.IOException; import java.io.RandomAccessFile; +import java.nio.charset.Charset; import java.nio.charset.StandardCharsets; import java.nio.file.DirectoryStream; import java.nio.file.FileSystem; @@ -49,9 +50,11 @@ import org.infinity.gui.menu.BrowserMenuBar; import org.infinity.resource.key.ResourceEntry; import org.infinity.resource.key.ResourceTreeModel; +import org.infinity.util.CharsetDetector; import org.infinity.util.DataString; import org.infinity.util.DebugTimer; import org.infinity.util.Logger; +import org.infinity.util.Misc; import org.infinity.util.Platform; import org.infinity.util.Table2da; import org.infinity.util.Table2daCache; @@ -288,6 +291,11 @@ public enum Key { * generated on first call of {@code getEquippedAppearanceMap()}. */ GET_GAME_EQUIPPED_APPEARANCES, + /** + * Property: ({@code String}) The autodetected character set used to encode or decode strings in the game. + * Can be overridden by NI's preferences. + */ + GET_GAME_CHARSET, /** Property: ({@code Boolean}) Has current game been enhanced by TobEx? */ IS_GAME_TOBEX, /** Property: ({@code Boolean}) Has current game been enhanced by EEex? */ @@ -939,6 +947,21 @@ public static Path getChitinKey() { return (ret instanceof Path) ? (Path) ret : null; } + /** + * Returns the default character set used by the currently open game to encode or decode game strings. + * + *

+ * Note: Charset detection uses heuristics for the original games and may not be fully accurate. + * It falls back to {@code windows-1252} if the charset could not be autodetected. + *

+ * + * @return Character set as {@link Charset} object. + */ + public static Charset getDefaultCharset() { + final String retVal = getProperty(Key.GET_GAME_CHARSET); + return Charset.forName((retVal != null) ? retVal : Misc.CHARSET_DEFAULT.name()); + } + /** * Updates language-related Properties with the specified game language. (Enhanced Editions only) * @@ -2371,6 +2394,9 @@ private void initFeatures() { Game game = getGame(); Engine engine = getEngine(); + // Autodetect default charset used by game strings + addEntry(Key.GET_GAME_CHARSET, Type.STRING, CharsetDetector.guessCharset(true)); + // Are Kits supported? addEntry(Key.IS_SUPPORTED_KITS, Type.BOOLEAN, (engine == Engine.BG2 || engine == Engine.IWD2 || engine == Engine.EE)); diff --git a/src/org/infinity/resource/bcs/BafResource.java b/src/org/infinity/resource/bcs/BafResource.java index 483395ce3..81bbe0637 100644 --- a/src/org/infinity/resource/bcs/BafResource.java +++ b/src/org/infinity/resource/bcs/BafResource.java @@ -352,9 +352,9 @@ public JComponent makeViewer(ViewableContainer container) { @Override public void write(OutputStream os) throws IOException { if (sourceText == null) { - StreamUtils.writeString(os, text, text.length()); + StreamUtils.writeString(os, text, text.length(), Profile.getDefaultCharset()); } else { - sourceText.write(new OutputStreamWriter(os)); + sourceText.write(new OutputStreamWriter(os, Profile.getDefaultCharset())); } } diff --git a/src/org/infinity/resource/bcs/BcsResource.java b/src/org/infinity/resource/bcs/BcsResource.java index 4be23cc6e..3adbf093e 100644 --- a/src/org/infinity/resource/bcs/BcsResource.java +++ b/src/org/infinity/resource/bcs/BcsResource.java @@ -672,9 +672,9 @@ public JComponent makeViewer(ViewableContainer container) { @Override public void write(OutputStream os) throws IOException { if (codeText == null) { - StreamUtils.writeString(os, text, text.length()); + StreamUtils.writeString(os, text, text.length(), Profile.getDefaultCharset()); } else { - StreamUtils.writeString(os, codeText.getText(), codeText.getText().length()); + StreamUtils.writeString(os, codeText.getText(), codeText.getText().length(), Profile.getDefaultCharset()); } } diff --git a/src/org/infinity/resource/cre/CreResource.java b/src/org/infinity/resource/cre/CreResource.java index b7b50d489..cf02b0929 100644 --- a/src/org/infinity/resource/cre/CreResource.java +++ b/src/org/infinity/resource/cre/CreResource.java @@ -558,13 +558,13 @@ public static void addScriptName(Map> scriptNames, Re if (signature.equalsIgnoreCase("CRE ")) { String version = StreamUtils.readString(buffer, offset + 4, 4); if (version.equalsIgnoreCase("V1.0")) { - scriptName = StreamUtils.readString(buffer, offset + 640, 32); + scriptName = StreamUtils.readString(buffer, offset + 640, 32, Profile.getDefaultCharset()); } else if (version.equalsIgnoreCase("V1.1") || version.equalsIgnoreCase("V1.2")) { - scriptName = StreamUtils.readString(buffer, offset + 804, 32); + scriptName = StreamUtils.readString(buffer, offset + 804, 32, Profile.getDefaultCharset()); } else if (version.equalsIgnoreCase("V2.2")) { - scriptName = StreamUtils.readString(buffer, offset + 916, 32); + scriptName = StreamUtils.readString(buffer, offset + 916, 32, Profile.getDefaultCharset()); } else if (version.equalsIgnoreCase("V9.0") || version.equalsIgnoreCase("V9.1")) { - scriptName = StreamUtils.readString(buffer, offset + 744, 32); + scriptName = StreamUtils.readString(buffer, offset + 744, 32, Profile.getDefaultCharset()); } if (scriptName.isEmpty() || scriptName.equalsIgnoreCase("None")) { return; @@ -712,7 +712,7 @@ public static String getSearchString(InputStream is) throws IOException { String sig = StreamUtils.readString(is, 4); is.skip(4); if (sig.equals("CHR ")) { - retVal = StreamUtils.readString(is, 32); + retVal = StreamUtils.readString(is, 32, Profile.getDefaultCharset()); } else { final int strrefName = StreamUtils.readInt(is); final int strrefShortName = StreamUtils.readInt(is); diff --git a/src/org/infinity/resource/dlg/AbstractCode.java b/src/org/infinity/resource/dlg/AbstractCode.java index 5e42cee6d..ab6d84fec 100644 --- a/src/org/infinity/resource/dlg/AbstractCode.java +++ b/src/org/infinity/resource/dlg/AbstractCode.java @@ -41,6 +41,7 @@ import org.infinity.icon.Icons; import org.infinity.resource.AbstractStruct; import org.infinity.resource.AddRemovable; +import org.infinity.resource.Profile; import org.infinity.resource.StructEntry; import org.infinity.resource.bcs.Compiler; import org.infinity.resource.bcs.ScriptMessage; @@ -78,7 +79,7 @@ protected AbstractCode(String name) { protected AbstractCode(ByteBuffer buffer, int offset, String name) { super(offset, 8, name); read(buffer, offset); - this.text = (len.getValue() > 0) ? StreamUtils.readString(buffer, off.getValue(), len.getValue()) : ""; + this.text = (len.getValue() > 0) ? StreamUtils.readString(buffer, off.getValue(), len.getValue(), Profile.getDefaultCharset()) : ""; } // --------------------- Begin Interface ActionListener --------------------- @@ -322,7 +323,7 @@ public void addFlatList(List flatList) { flatList.add(off); flatList.add(len); try { - TextString ts = new TextString(StreamUtils.getByteBuffer(text.getBytes()), 0, len.getValue(), DLG_CODE_TEXT); + TextString ts = new TextString(StreamUtils.getByteBuffer(text.getBytes(Profile.getDefaultCharset())), 0, len.getValue(), DLG_CODE_TEXT); ts.setOffset(off.getValue()); flatList.add(ts); } catch (Exception e) { @@ -345,7 +346,7 @@ public int updateOffset(int offs) { } public void writeString(OutputStream os) throws IOException { - StreamUtils.writeString(os, text, len.getValue()); + StreamUtils.writeString(os, text, len.getValue(), Profile.getDefaultCharset()); } private void highlightLine(int linenr) { diff --git a/src/org/infinity/resource/text/PlainTextResource.java b/src/org/infinity/resource/text/PlainTextResource.java index 1a4461325..ebd163c1c 100644 --- a/src/org/infinity/resource/text/PlainTextResource.java +++ b/src/org/infinity/resource/text/PlainTextResource.java @@ -582,9 +582,9 @@ public JComponent makeViewer(ViewableContainer container) { @Override public void write(OutputStream os) throws IOException { if (editor == null) { - StreamUtils.writeString(os, text, text.length()); + StreamUtils.writeString(os, text, text.length(), Profile.getDefaultCharset()); } else { - editor.write(new OutputStreamWriter(os)); + editor.write(new OutputStreamWriter(os, Profile.getDefaultCharset())); } } diff --git a/src/org/infinity/util/Misc.java b/src/org/infinity/util/Misc.java index c7984ab7f..6cafbb0df 100644 --- a/src/org/infinity/util/Misc.java +++ b/src/org/infinity/util/Misc.java @@ -126,15 +126,6 @@ public static Preferences getPrefs(String className) { return Preferences.userRoot().node(Misc.prefsNodeName(className)); } - /** - * Returns a default charset depending on the current game type. - * - * @return {@link #CHARSET_UTF8} for Enhanced Edition games, {@link #CHARSET_DEFAULT} otherwise. - */ - public static Charset getDefaultCharset() { - return Profile.isEnhancedEdition() ? CHARSET_UTF8 : CHARSET_DEFAULT; - } - /** * A convenience method that attempts to return the charset specified by the given name or the next best match * depending on the current game type. @@ -143,7 +134,7 @@ public static Charset getDefaultCharset() { * @return The desired charset if successful, a game-specific default charset otherwise. */ public static Charset getCharsetFrom(String charsetName) { - return getCharsetFrom(charsetName, getDefaultCharset()); + return getCharsetFrom(charsetName, Profile.getDefaultCharset()); } /** diff --git a/src/org/infinity/util/StringTable.java b/src/org/infinity/util/StringTable.java index 3013ee7e8..25e53e92c 100644 --- a/src/org/infinity/util/StringTable.java +++ b/src/org/infinity/util/StringTable.java @@ -108,7 +108,7 @@ public static Charset getCharset() { setCharset(BrowserMenuBar.getInstance().getOptions().getSelectedCharset()); } catch (Throwable t) { // returns a temporary value if BrowserMenuBar has not yet been initialized - return Profile.isEnhancedEdition() ? Misc.CHARSET_UTF8 : Misc.CHARSET_DEFAULT; + return Profile.getDefaultCharset(); } } return charset;