Utf-8 check, C++ initial version

engine/src/cmd/basecomputer.cpp

@@ -92,7 +92,6 @@ using namespace XMLSupport;
 //end for directory thing
 extern const char *DamagedCategory;
 
-
 int BaseComputer:: dirty = 0;
 
 static GFXColor UnsaturatedColor( float r, float g, float b, float a = 1.0f )
@@ -5980,6 +5979,10 @@ bool BaseComputer::actionConfirmedLoadGame()
     if (desc) {
         std::string tmp = desc->text();
         if (tmp.length() > 0) {
+            if (!isUtf8SaveGame(tmp)) {
+                showAlert( tmp + " is not UTF-8, convert it before loading" );
+                return true;
+            }
             Cockpit *cockpit = player ? _Universe->isPlayerStarship( player ) : 0;
             if (player && cockpit) {
                 UniverseUtil::showSplashScreen( "" );

engine/src/cmd/script/director_generic.cpp

@@ -327,6 +327,11 @@ static void saveStringListPy( int playernum, string mykey, vector< string > name
     saveStringList(playernum, mykey, names);
 }
 
+static bool isUtf8SaveGamePy( string savegame )
+{
+    return isUtf8SaveGame(savegame);
+}
+
 
 PYTHON_BEGIN_MODULE( Director )
 PYTHON_BEGIN_INHERIT_CLASS( Director, pythonMission, PythonMissionBaseClass, "Mission" )
@@ -348,6 +353,7 @@ PYTHON_DEFINE_GLOBAL( Director, &eraseSaveStringPy, "eraseSaveString" );
 PYTHON_DEFINE_GLOBAL( Director, &clearSaveStringPy, "clearSaveString" );
 PYTHON_DEFINE_GLOBAL( Director, &loadStringListPy, "loadStringList" );
 PYTHON_DEFINE_GLOBAL( Director, &saveStringListPy, "saveStringList" );
+PYTHON_DEFINE_GLOBAL( Director, &isUtf8SaveGamePy, "isUtf8SaveGame" );
 PYTHON_END_MODULE( Director )
 
 void InitDirector()

engine/src/savegame.cpp

@@ -44,6 +44,12 @@
 #include "universe.h"
 #include "options.h"
 
+#include <iostream>
+#include <fstream>
+#include <iterator>
+#include <vector>
+
+typedef unsigned char BYTE;
 
 
 using namespace VSFileSystem;
@@ -174,6 +180,105 @@ void SaveFileCopy( const char *src, const char *dst )
     }
 }
 
+class Utf8Checker {
+public:
+    /*
+     * From https://helloacm.com/how-to-validate-utf-8-encoding-the-simple-utf-8-validation-algorithm/
+     */
+    bool validUtf8(const std::vector<BYTE> &data) {
+        for (long unsigned int i = 0; i < data.size(); ++ i) {
+            // 0xxxxxxx
+            int bit1 = (data[i] >> 7) & 1;
+            if (bit1 == 0) continue;
+            // 110xxxxx 10xxxxxx
+            int bit2 = (data[i] >> 6) & 1;
+            if (bit2 == 0) return false;
+            // 11
+            int bit3 = (data[i] >> 5) & 1;
+            if (bit3 == 0) {
+                // 110xxxxx 10xxxxxx
+                if ((++ i) < data.size()) {
+                    if (is10x(data[i])) {
+                        continue;
+                    }
+                    return false;
+                } else {
+                    return false;
+                }
+            }
+            int bit4 = (data[i] >> 4) & 1;
+            if (bit4 == 0) {
+                // 1110xxxx 10xxxxxx 10xxxxxx
+                if (i + 2 < data.size()) {
+                    if (is10x(data[i + 1]) && is10x(data[i + 2])) {
+                        i += 2;
+                        continue;
+                    }
+                    return false;
+                } else {
+                    return false;
+                }
+            }
+            int bit5 = (data[i] >> 3) & 1;
+            if (bit5 == 1) return false;
+            if (i + 3 < data.size()) {
+                if (is10x(data[i + 1]) && is10x(data[i + 2]) && is10x(data[i + 3])) {
+                    i += 3;
+                    continue;
+                }
+                return false;
+            } else {
+                return false;
+            }
+        }
+        return true;
+    }
+private:
+    inline bool is10x(int a) {
+        int bit1 = (a >> 7) & 1;
+        int bit2 = (a >> 6) & 1;
+        return (bit1 == 1) && (bit2 == 0);
+    }
+};
+
+std::vector<BYTE> readFile(std::string filename)
+/*
+ * From https://stackoverflow.com/a/21802936
+ */
+{
+    std::ifstream file(filename, std::ios::binary);
+    file.unsetf(std::ios::skipws);
+    std::streampos fileSize;
+
+    file.seekg(0, std::ios::end);
+    fileSize = file.tellg();
+    file.seekg(0, std::ios::beg);
+
+    std::vector<BYTE> vec;
+    vec.reserve(fileSize);
+
+    std::copy(std::istream_iterator<BYTE>(file),
+          std::istream_iterator<BYTE>(),
+          std::back_inserter(vec));
+
+    file.close();
+
+    return vec;
+}
+
+bool isUtf8SaveGame(std::string filename)
+{
+    std::string path = GetSaveDir() + filename;
+    std::vector<BYTE> savegame = readFile(path);
+    Utf8Checker* check = new Utf8Checker;
+    if (check->validUtf8(savegame)) {
+        return true;
+    } else {
+        BOOST_LOG_TRIVIAL(fatal) << boost::format("ERROR: save file %1% is not UTF-8") % path;
+        return false;
+    }
+}
+
 class MissionStringDat
 {
 public:

engine/src/savegame.h

@@ -155,5 +155,6 @@ const std::string& GetCurrentSaveGame();
 std::string SetCurrentSaveGame( std::string newname );
 const std::string& GetSaveDir();
 void CopySavedShips( std::string filename, int player_num, const std::vector< std::string > &starships, bool load );
+bool isUtf8SaveGame(std::string filename);
 #endif