From eedf399d9dcad2e4468e6735178c74b139cdcc95 Mon Sep 17 00:00:00 2001
From: Geoff Hutchison <geoff.hutchison@gmail.com>
Date: Tue, 31 Dec 2024 14:36:30 -0500
Subject: [PATCH] Read UTF-16 files through Qt decoding to a temporary file

Fixes https://github.com/OpenChemistry/avogadrolibs/issues/1689

Signed-off-by: Geoff Hutchison <geoff.hutchison@gmail.com>
---
 avogadro/backgroundfileformat.cpp | 63 ++++++++++++++++++++++++++++---
 1 file changed, 58 insertions(+), 5 deletions(-)

diff --git a/avogadro/backgroundfileformat.cpp b/avogadro/backgroundfileformat.cpp
index 5e4890f3..9318b5ea 100644
--- a/avogadro/backgroundfileformat.cpp
+++ b/avogadro/backgroundfileformat.cpp
@@ -7,11 +7,18 @@
 
 #include <avogadro/io/fileformat.h>
 
+#include <QtCore/QFile>
+#include <QtCore/QTemporaryDir>
+#include <QtCore/QTextStream>
+
 namespace Avogadro {
 
 BackgroundFileFormat::BackgroundFileFormat(Io::FileFormat* format,
                                            QObject* aparent)
-  : QObject(aparent), m_format(format), m_molecule(nullptr), m_success(false)
+  : QObject(aparent)
+  , m_format(format)
+  , m_molecule(nullptr)
+  , m_success(false)
 {
 }
 
@@ -35,8 +42,54 @@ void BackgroundFileFormat::read()
     m_error = tr("No file name set in BackgroundFileFormat!");
 
   if (m_error.isEmpty()) {
-    m_success = m_format->readFile(m_fileName.toLocal8Bit().data(),
-                                   *m_molecule);
+    // sometimes we hit UTF-16 files, so we need to convert them to UTF-8
+    // first check whether the file is UTF-16
+    QFile file(m_fileName);
+    QTextStream in(&file);
+    QString text;
+    bool isUTF16 = false;
+    if (file.open(QIODevice::ReadOnly)) {
+      QByteArray data = file.read(2);
+      // look for a byte-order mark
+      if ((data.size() == 2 && data[0] == '\xff' && data[1] == '\xfe') ||
+          (data.size() == 2 && data[0] == '\xfe' && data[1] == '\xff')) {
+        // UTF-16, read the file and let QString handle decoding
+        isUTF16 = true;
+        file.close();
+        file.open(QIODevice::ReadOnly | QIODevice::Text);
+        in.setCodec("UTF-16");
+        text = in.readAll();
+        file.close();
+      }
+    }
+
+    if (!isUTF16)
+      m_success =
+        m_format->readFile(m_fileName.toLocal8Bit().data(), *m_molecule);
+    else {
+      // write it to a temporary file and we'll read it back in
+      // some formats (like the generic output) need a file
+      // not just a string bugger
+
+      // first, we need the *name* of the file, not the full path
+      // because we're going to save a copy in a temp directory
+      QTemporaryDir tempDir;
+      QString tempFileName = tempDir.filePath(QFileInfo(m_fileName).fileName());
+      QFile tempFile(tempFileName);
+      if (tempFile.open(QIODevice::WriteOnly | QIODevice::Text)) {
+        QTextStream out(&tempFile);
+        // set it to UTF-8
+        out.setCodec("UTF-8");
+        out << text;
+        out.flush();
+        tempFile.close();
+        m_success =
+          m_format->readFile(tempFileName.toLocal8Bit().data(), *m_molecule);
+        tempFile.remove();
+      } else // try just reading the string
+        m_success =
+          m_format->readString(text.toLocal8Bit().data(), *m_molecule);
+    }
 
     if (!m_success)
       m_error = QString::fromStdString(m_format->error());
@@ -60,8 +113,8 @@ void BackgroundFileFormat::write()
     m_error = tr("No file name set in BackgroundFileFormat!");
 
   if (m_error.isEmpty()) {
-    m_success = m_format->writeFile(m_fileName.toLocal8Bit().data(),
-                                    *m_molecule);
+    m_success =
+      m_format->writeFile(m_fileName.toLocal8Bit().data(), *m_molecule);
 
     if (!m_success)
       m_error = QString::fromStdString(m_format->error());