Improve handling of strings with non-ASCII characters in game resources

src/org/infinity/datatype/TextString.java

@@ -66,7 +66,7 @@ public boolean update(Object value) {
   @Override
   public void write(OutputStream os) throws IOException {
     if (text != null) {
-      byte[] buf = text.getBytes(Misc.CHARSET_DEFAULT);
+      byte[] buf = text.getBytes(charset);
       buffer.position(0);
       buffer.put(buf, 0, Math.min(buf.length, buffer.limit()));
       while (buffer.remaining() > 0) {

src/org/infinity/gui/GameProperties.java

@@ -227,6 +227,11 @@ private void init() {
       listControls.add(Couple.with(l, tf));
     }
 
+    // Entry: Default charset
+    l = new JLabel("Default character set:");
+    tf = createReadOnlyField(Profile.getDefaultCharset().displayName(), true);
+    listControls.add(Couple.with(l,  tf));
+
     // Entry: Use female TLK file
     l = new JLabel("Uses female TLK file:");
     tf = createReadOnlyField(Boolean.toString((Profile.getProperty(Profile.Key.GET_GAME_DIALOGF_FILE) != null)), true);

src/org/infinity/gui/menu/OptionsMenuItem.java

@@ -410,9 +410,9 @@ public boolean showPreferencesDialog(Window owner) {
   }
 
   /** Attempts to determine the correct charset for the current game. */
-  public String charsetName(String charset, boolean detect) {
+  public String charsetName(String charset) {
     if (DEFAULT_CHARSET.equalsIgnoreCase(charset)) {
-      charset = CharsetDetector.guessCharset(detect);
+      charset = Profile.getDefaultCharset().name();
     } else {
       charset = CharsetDetector.setCharset(charset);
     }
@@ -856,7 +856,7 @@ public boolean isCharsetAvailable(String charset) {
 
   /** Returns the character encoding of the string table. */
   public String getSelectedCharset() {
-    return charsetName(AppOption.TLK_CHARSET_TYPE.getStringValue(), true);
+    return charsetName(AppOption.TLK_CHARSET_TYPE.getStringValue());
   }
 
   /** Returns the currently selected game language. Returns empty string on autodetect. */
@@ -941,7 +941,7 @@ private void applyChanges(Collection<AppOption> options) {
           final String csName = option.getStringValue();
           if (csName != null) {
             CharsetDetector.clearCache();
-            StringTable.setCharset(charsetName(csName, true));
+            StringTable.setCharset(charsetName(csName));
             messages.add("TLK Character Encoding: " + csName);
             // enforce re-reading strings
             refresh = true;

src/org/infinity/resource/Profile.java

@@ -11,6 +11,7 @@
 import java.io.File;
 import java.io.IOException;
 import java.io.RandomAccessFile;
+import java.nio.charset.Charset;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.DirectoryStream;
 import java.nio.file.FileSystem;
@@ -49,9 +50,11 @@
 import org.infinity.gui.menu.BrowserMenuBar;
 import org.infinity.resource.key.ResourceEntry;
 import org.infinity.resource.key.ResourceTreeModel;
+import org.infinity.util.CharsetDetector;
 import org.infinity.util.DataString;
 import org.infinity.util.DebugTimer;
 import org.infinity.util.Logger;
+import org.infinity.util.Misc;
 import org.infinity.util.Platform;
 import org.infinity.util.Table2da;
 import org.infinity.util.Table2daCache;
@@ -288,6 +291,11 @@ public enum Key {
      * generated on first call of {@code getEquippedAppearanceMap()}.
      */
     GET_GAME_EQUIPPED_APPEARANCES,
+    /**
+     * Property: ({@code String}) The autodetected character set used to encode or decode strings in the game.
+     * Can be overridden by NI's preferences.
+     */
+    GET_GAME_CHARSET,
     /** Property: ({@code Boolean}) Has current game been enhanced by TobEx? */
     IS_GAME_TOBEX,
     /** Property: ({@code Boolean}) Has current game been enhanced by EEex? */
@@ -939,6 +947,21 @@ public static Path getChitinKey() {
     return (ret instanceof Path) ? (Path) ret : null;
   }
 
+  /**
+   * Returns the default character set used by the currently open game to encode or decode game strings.
+   *
+   * <p>
+   * <b>Note:</b> Charset detection uses heuristics for the original games and may not be fully accurate.
+   * It falls back to {@code windows-1252} if the charset could not be autodetected.
+   * </p>
+   *
+   * @return Character set as {@link Charset} object.
+   */
+  public static Charset getDefaultCharset() {
+    final String retVal = getProperty(Key.GET_GAME_CHARSET);
+    return Charset.forName((retVal != null) ? retVal : Misc.CHARSET_DEFAULT.name());
+  }
+
   /**
    * Updates language-related Properties with the specified game language. (Enhanced Editions only)
    *
@@ -2371,6 +2394,9 @@ private void initFeatures() {
     Game game = getGame();
     Engine engine = getEngine();
 
+    // Autodetect default charset used by game strings
+    addEntry(Key.GET_GAME_CHARSET, Type.STRING, CharsetDetector.guessCharset(true));
+
     // Are Kits supported?
     addEntry(Key.IS_SUPPORTED_KITS, Type.BOOLEAN,
         (engine == Engine.BG2 || engine == Engine.IWD2 || engine == Engine.EE));

src/org/infinity/resource/bcs/BafResource.java

@@ -352,9 +352,9 @@ public JComponent makeViewer(ViewableContainer container) {
   @Override
   public void write(OutputStream os) throws IOException {
     if (sourceText == null) {
-      StreamUtils.writeString(os, text, text.length());
+      StreamUtils.writeString(os, text, text.length(), Profile.getDefaultCharset());
     } else {
-      sourceText.write(new OutputStreamWriter(os));
+      sourceText.write(new OutputStreamWriter(os, Profile.getDefaultCharset()));
     }
   }
 

src/org/infinity/resource/bcs/BcsResource.java

@@ -672,9 +672,9 @@ public JComponent makeViewer(ViewableContainer container) {
   @Override
   public void write(OutputStream os) throws IOException {
     if (codeText == null) {
-      StreamUtils.writeString(os, text, text.length());
+      StreamUtils.writeString(os, text, text.length(), Profile.getDefaultCharset());
     } else {
-      StreamUtils.writeString(os, codeText.getText(), codeText.getText().length());
+      StreamUtils.writeString(os, codeText.getText(), codeText.getText().length(), Profile.getDefaultCharset());
     }
   }
 

src/org/infinity/resource/cre/CreResource.java

@@ -558,13 +558,13 @@ public static void addScriptName(Map<String, Set<ResourceEntry>> scriptNames, Re
       if (signature.equalsIgnoreCase("CRE ")) {
         String version = StreamUtils.readString(buffer, offset + 4, 4);
         if (version.equalsIgnoreCase("V1.0")) {
-          scriptName = StreamUtils.readString(buffer, offset + 640, 32);
+          scriptName = StreamUtils.readString(buffer, offset + 640, 32, Profile.getDefaultCharset());
         } else if (version.equalsIgnoreCase("V1.1") || version.equalsIgnoreCase("V1.2")) {
-          scriptName = StreamUtils.readString(buffer, offset + 804, 32);
+          scriptName = StreamUtils.readString(buffer, offset + 804, 32, Profile.getDefaultCharset());
         } else if (version.equalsIgnoreCase("V2.2")) {
-          scriptName = StreamUtils.readString(buffer, offset + 916, 32);
+          scriptName = StreamUtils.readString(buffer, offset + 916, 32, Profile.getDefaultCharset());
         } else if (version.equalsIgnoreCase("V9.0") || version.equalsIgnoreCase("V9.1")) {
-          scriptName = StreamUtils.readString(buffer, offset + 744, 32);
+          scriptName = StreamUtils.readString(buffer, offset + 744, 32, Profile.getDefaultCharset());
         }
         if (scriptName.isEmpty() || scriptName.equalsIgnoreCase("None")) {
           return;
@@ -712,7 +712,7 @@ public static String getSearchString(InputStream is) throws IOException {
     String sig = StreamUtils.readString(is, 4);
     is.skip(4);
     if (sig.equals("CHR ")) {
-      retVal = StreamUtils.readString(is, 32);
+      retVal = StreamUtils.readString(is, 32, Profile.getDefaultCharset());
     } else {
       final int strrefName = StreamUtils.readInt(is);
       final int strrefShortName = StreamUtils.readInt(is);

src/org/infinity/resource/dlg/AbstractCode.java

@@ -41,6 +41,7 @@
 import org.infinity.icon.Icons;
 import org.infinity.resource.AbstractStruct;
 import org.infinity.resource.AddRemovable;
+import org.infinity.resource.Profile;
 import org.infinity.resource.StructEntry;
 import org.infinity.resource.bcs.Compiler;
 import org.infinity.resource.bcs.ScriptMessage;
@@ -78,7 +79,7 @@ protected AbstractCode(String name) {
   protected AbstractCode(ByteBuffer buffer, int offset, String name) {
     super(offset, 8, name);
     read(buffer, offset);
-    this.text = (len.getValue() > 0) ? StreamUtils.readString(buffer, off.getValue(), len.getValue()) : "";
+    this.text = (len.getValue() > 0) ? StreamUtils.readString(buffer, off.getValue(), len.getValue(), Profile.getDefaultCharset()) : "";
   }
 
   // --------------------- Begin Interface ActionListener ---------------------
@@ -322,7 +323,7 @@ public void addFlatList(List<StructEntry> flatList) {
     flatList.add(off);
     flatList.add(len);
     try {
-      TextString ts = new TextString(StreamUtils.getByteBuffer(text.getBytes()), 0, len.getValue(), DLG_CODE_TEXT);
+      TextString ts = new TextString(StreamUtils.getByteBuffer(text.getBytes(Profile.getDefaultCharset())), 0, len.getValue(), DLG_CODE_TEXT);
       ts.setOffset(off.getValue());
       flatList.add(ts);
     } catch (Exception e) {
@@ -345,7 +346,7 @@ public int updateOffset(int offs) {
   }
 
   public void writeString(OutputStream os) throws IOException {
-    StreamUtils.writeString(os, text, len.getValue());
+    StreamUtils.writeString(os, text, len.getValue(), Profile.getDefaultCharset());
   }
 
   private void highlightLine(int linenr) {

src/org/infinity/resource/text/PlainTextResource.java

@@ -582,9 +582,9 @@ public JComponent makeViewer(ViewableContainer container) {
   @Override
   public void write(OutputStream os) throws IOException {
     if (editor == null) {
-      StreamUtils.writeString(os, text, text.length());
+      StreamUtils.writeString(os, text, text.length(), Profile.getDefaultCharset());
     } else {
-      editor.write(new OutputStreamWriter(os));
+      editor.write(new OutputStreamWriter(os, Profile.getDefaultCharset()));
     }
   }
 

src/org/infinity/util/Misc.java

@@ -126,15 +126,6 @@ public static Preferences getPrefs(String className) {
     return Preferences.userRoot().node(Misc.prefsNodeName(className));
   }
 
-  /**
-   * Returns a default charset depending on the current game type.
-   *
-   * @return {@link #CHARSET_UTF8} for Enhanced Edition games, {@link #CHARSET_DEFAULT} otherwise.
-   */
-  public static Charset getDefaultCharset() {
-    return Profile.isEnhancedEdition() ? CHARSET_UTF8 : CHARSET_DEFAULT;
-  }
-
   /**
    * A convenience method that attempts to return the charset specified by the given name or the next best match
    * depending on the current game type.
@@ -143,7 +134,7 @@ public static Charset getDefaultCharset() {
    * @return The desired charset if successful, a game-specific default charset otherwise.
    */
   public static Charset getCharsetFrom(String charsetName) {
-    return getCharsetFrom(charsetName, getDefaultCharset());
+    return getCharsetFrom(charsetName, Profile.getDefaultCharset());
   }
 
   /**

src/org/infinity/util/StringTable.java

@@ -108,7 +108,7 @@ public static Charset getCharset() {
         setCharset(BrowserMenuBar.getInstance().getOptions().getSelectedCharset());
       } catch (Throwable t) {
         // returns a temporary value if BrowserMenuBar has not yet been initialized
-        return Profile.isEnhancedEdition() ? Misc.CHARSET_UTF8 : Misc.CHARSET_DEFAULT;
+        return Profile.getDefaultCharset();
       }
     }
     return charset;