Script: Add Unicode support

If the script file is UTF-8 encoded, all Lua strings are converted to/from as if they were UTF-8. Otherwise things continue to work as they have until now. Additionally, UTF-8 scripts cannot use deprecated features at all (PROPERTIES, GetStringValue).

Library/MeasureScript.cpp

@@ -112,18 +112,12 @@ void MeasureScript::ReadOptions(ConfigParser& parser, const WCHAR* section)
 		{
 			UninitializeLuaScript();
 
-			lua_State* L = LuaManager::GetState();
 			if (m_LuaScript.Initialize(scriptFile))
 			{
 				bool hasInitializeFunction = m_LuaScript.IsFunction(g_InitializeFunctionName);
 				m_HasUpdateFunction = m_LuaScript.IsFunction(g_UpdateFunctionName);
-				m_HasGetStringFunction = m_LuaScript.IsFunction(g_GetStringFunctionName);  // For backwards compatbility
-
-				if (m_HasGetStringFunction)
-				{
-					LogWarningF(this, L"Script: Using deprecated GetStringValue()");
-				}
 
+				lua_State* L = m_LuaScript.GetState();
 				lua_rawgeti(L, LUA_GLOBALSINDEX, m_LuaScript.GetRef());
 
 				*(MeterWindow**)lua_newuserdata(L, sizeof(MeterWindow*)) = m_MeterWindow;
@@ -136,28 +130,35 @@ void MeasureScript::ReadOptions(ConfigParser& parser, const WCHAR* section)
 				lua_setmetatable(L, -2);
 				lua_setfield(L, -2, "SELF");
 
-				// For backwards compatibility
+				if (!m_LuaScript.IsUnicode())
-				lua_getfield(L, -1, "PROPERTIES");
-				if (lua_isnil(L, -1) == 0)
 				{
-					lua_pushnil(L);
+					// For backwards compatibility.
-					
+
-					// Look in the table for values to read from the section
+					m_HasGetStringFunction = m_LuaScript.IsFunction(g_GetStringFunctionName);
-					while (lua_next(L, -2))
+					if (m_HasGetStringFunction)
 					{
-						lua_pop(L, 1);
+						LogWarningF(this, L"Script: Using deprecated GetStringValue()");
-						const char* strKey = lua_tostring(L, -1);
+					}
 
-						std::wstring wstrKey = StringUtil::Widen(strKey);
+					lua_getfield(L, -1, "PROPERTIES");
-						const std::wstring& wstrValue = parser.ReadString(section, wstrKey.c_str(), L"");
+					if (lua_isnil(L, -1) == 0)
-
+					{
-						if (!wstrValue.empty())
+						lua_pushnil(L);
+					
+						// Look in the table for values to read from the section
+						while (lua_next(L, -2))
 						{
-							std::string strStrVal = StringUtil::Narrow(wstrValue);
+							lua_pop(L, 1);
-							const char* strValue = strStrVal.c_str();
+							const char* strKey = lua_tostring(L, -1);
-
+							const std::wstring wstrKey = StringUtil::Widen(strKey);
-							lua_pushstring(L, strValue);
+							const std::wstring& wstrValue =
-							lua_setfield(L, -3, strKey);
+								parser.ReadString(section, wstrKey.c_str(), L"");
+							if (!wstrValue.empty())
+							{
+								const std::string strStrVal = StringUtil::Narrow(wstrValue);
+								lua_pushstring(L, strStrVal.c_str());
+								lua_setfield(L, -3, strKey);
+							}
 						}
 					}
 				}
@@ -193,8 +194,7 @@ void MeasureScript::ReadOptions(ConfigParser& parser, const WCHAR* section)
 */
 void MeasureScript::Command(const std::wstring& command)
 {
-	std::string str = StringUtil::Narrow(command);
+	m_LuaScript.RunString(command);
-	m_LuaScript.RunString(str.c_str());
 }
 
 //static void stackDump(lua_State *L)

Library/lua/LuaManager.cpp

@@ -24,6 +24,8 @@
 int LuaManager::c_RefCount = 0;
 lua_State* LuaManager::c_State = 0;
 
+bool LuaManager::s_UnicodeState = false;
+
 void LuaManager::Initialize()
 {
 	if (c_State == nullptr)
@@ -57,8 +59,9 @@ void LuaManager::Finalize()
 	}
 }
 
-void LuaManager::ReportErrors(lua_State* L, const std::wstring& file)
+void LuaManager::ReportErrors(const std::wstring& file)
 {
+	lua_State* L = c_State;
 	const char* error = lua_tostring(L, -1);
 	lua_pop(L, 1);
 
@@ -70,25 +73,31 @@ void LuaManager::ReportErrors(lua_State* L, const std::wstring& file)
 	}
 
 	std::wstring str(file, file.find_last_of(L'\\') + 1);
-	str += StringUtil::Widen(error);
+	str += s_UnicodeState ? StringUtil::WidenUTF8(error) : StringUtil::Widen(error);
 	LogErrorF(L"Script: %s", str.c_str());
 }
 
-void LuaManager::PushWide(lua_State* L, const WCHAR* str)
+void LuaManager::PushWide(const WCHAR* str)
 {
-	const std::string narrowStr = StringUtil::Narrow(str);
+	lua_State* L = c_State;
+	const std::string narrowStr = s_UnicodeState ?
+		StringUtil::NarrowUTF8(str) : StringUtil::Narrow(str);
 	lua_pushlstring(L, narrowStr.c_str(), narrowStr.length());
 }
 
-void LuaManager::PushWide(lua_State* L, const std::wstring& str)
+void LuaManager::PushWide(const std::wstring& str)
 {
-	const std::string narrowStr = StringUtil::Narrow(str);
+	lua_State* L = c_State;
+	const std::string narrowStr = s_UnicodeState ?
+		StringUtil::NarrowUTF8(str) : StringUtil::Narrow(str);
 	lua_pushlstring(L, narrowStr.c_str(), narrowStr.length());
 }
 
-std::wstring LuaManager::ToWide(lua_State* L, int narg)
+std::wstring LuaManager::ToWide(int narg)
 {
+	lua_State* L = c_State;
 	size_t strLen = 0;
 	const char* str = lua_tolstring(L, narg, &strLen);
-	return StringUtil::Widen(str, (int)strLen);
+	return s_UnicodeState ?
+		StringUtil::WidenUTF8(str, (int)strLen) : StringUtil::Widen(str, (int)strLen);
 }

Library/lua/LuaManager.h

@@ -32,13 +32,13 @@ public:
 	static void Initialize();
 	static void Finalize();
 
-	static lua_State* GetState() { return c_State; }
+	static lua_State* GetState(bool unicode) { s_UnicodeState = unicode; return c_State; }
 
-	static void ReportErrors(lua_State* L, const std::wstring& file);
+	static void ReportErrors(const std::wstring& file);
 
-	static void PushWide(lua_State* L, const WCHAR* str);
+	static void PushWide(const WCHAR* str);
-	static void PushWide(lua_State* L, const std::wstring& str);
+	static void PushWide(const std::wstring& str);
-	static std::wstring ToWide(lua_State* L, int narg);
+	static std::wstring ToWide(int narg);
 
 protected:
 	static int c_RefCount;
@@ -50,6 +50,10 @@ private:
 	static void RegisterMeter(lua_State* L);
 	static void RegisterMeterWindow(lua_State* L);
 	static void RegisterMeterString(lua_State* L);
+
+	// If set true |true|, Lua strings converted to/from as if they were encoded in UTF-8. Otherwise
+	// Lua strings are treated as if they are encoded in the default system encoding.
+	static bool s_UnicodeState;
 };
 
 #endif

Library/lua/LuaScript.cpp

@@ -26,7 +26,8 @@
 **
 */
 LuaScript::LuaScript() :
-	m_Ref(LUA_NOREF)
+	m_Ref(LUA_NOREF),
+	m_Unicode(false)
 {
 }
 
@@ -43,29 +44,36 @@ bool LuaScript::Initialize(const std::wstring& scriptFile)
 {
 	assert(!IsInitialized());
 
-	lua_State* L = LuaManager::GetState();
-
-	// Load file into a buffer as luaL_loadfile does not support Unicode paths.
 	FILE* file = _wfopen(scriptFile.c_str(), L"rb");
-	if (!file)
+	if (!file) return false;
-	{
-		return false;
-	}
 
 	fseek(file, 0, SEEK_END);
-	long fileSize = ftell(file);
+	const long fileSize = ftell(file);
-
+	BYTE* fileData = new BYTE[fileSize];
-	char* fileData = new char[fileSize];
 	fseek(file, 0, SEEK_SET);
 	fread(fileData, fileSize, 1, file);
-
 	fclose(file);
 	file = nullptr;
 
-	int load = luaL_loadbuffer(L, fileData, fileSize, "");
+	// Has UTF8 BOM, so assume that data is already in UTF8.
+	char* scriptData = (char*)fileData;
+	int scriptDataSize = fileSize;
+
+	// Treat the script as Unicode if it has the UTF-8 BOM.
+	m_Unicode = fileSize > 3 && fileData[0] == 0xEF && fileData[1] == 0xBB && fileData[2] == 0xBF;
+	if (m_Unicode)
+	{
+		// Skip the BOM.
+		scriptData += 3;
+		scriptDataSize -= 3;
+	}
+
+	lua_State* L = GetState();
+	bool scriptLoaded = luaL_loadbuffer(L, scriptData, scriptDataSize, "") == 0;
+
 	delete [] fileData;
 
-	if (load == 0)
+	if (scriptLoaded)
 	{
 		// Create the table this script will reside in
 		lua_newtable(L);
@@ -100,13 +108,13 @@ bool LuaScript::Initialize(const std::wstring& scriptFile)
 		}
 		else
 		{
-			LuaManager::ReportErrors(L, scriptFile);
+			LuaManager::ReportErrors(scriptFile);
 			Uninitialize();
 		}
 	}
 	else
 	{
-		LuaManager::ReportErrors(L, scriptFile);
+		LuaManager::ReportErrors(scriptFile);
 	}
 
 	return false;
@@ -114,7 +122,7 @@ bool LuaScript::Initialize(const std::wstring& scriptFile)
 
 void LuaScript::Uninitialize()
 {
-	lua_State* L = LuaManager::GetState();
+	lua_State* L = GetState();
 
 	if (m_Ref != LUA_NOREF)
 	{
@@ -130,7 +138,7 @@ void LuaScript::Uninitialize()
 */
 bool LuaScript::IsFunction(const char* funcName)
 {
-	lua_State* L = LuaManager::GetState();
+	lua_State* L = GetState();
 	bool bExists = false;
 
 	if (IsInitialized())
@@ -156,7 +164,7 @@ bool LuaScript::IsFunction(const char* funcName)
 */
 void LuaScript::RunFunction(const char* funcName)
 {
-	lua_State* L = LuaManager::GetState();
+	lua_State* L = GetState();
 
 	if (IsInitialized())
 	{
@@ -168,7 +176,7 @@ void LuaScript::RunFunction(const char* funcName)
 
 		if (lua_pcall(L, 0, 0, 0))
 		{
-			LuaManager::ReportErrors(L, m_File);
+			LuaManager::ReportErrors(m_File);
 		}
 
 		lua_pop(L, 1);
@@ -181,7 +189,7 @@ void LuaScript::RunFunction(const char* funcName)
 */
 int LuaScript::RunFunctionWithReturn(const char* funcName, double& numValue, std::wstring& strValue)
 {
-	lua_State* L = LuaManager::GetState();
+	lua_State* L = GetState();
 	int type = LUA_TNIL;
 
 	if (IsInitialized())
@@ -194,7 +202,7 @@ int LuaScript::RunFunctionWithReturn(const char* funcName, double& numValue, std
 
 		if (lua_pcall(L, 0, 1, 0))
 		{
-			LuaManager::ReportErrors(L, m_File);
+			LuaManager::ReportErrors(m_File);
 			lua_pop(L, 1);
 		}
 		else
@@ -208,7 +216,8 @@ int LuaScript::RunFunctionWithReturn(const char* funcName, double& numValue, std
 			{
 				size_t strLen = 0;
 				const char* str = lua_tolstring(L, -1, &strLen);
-				strValue = StringUtil::Widen(str, (int)strLen);
+				strValue = m_Unicode ?
+					StringUtil::WidenUTF8(str, (int)strLen) : StringUtil::Widen(str, (int)strLen);
 				numValue = strtod(str, nullptr);
 			}
 
@@ -223,16 +232,19 @@ int LuaScript::RunFunctionWithReturn(const char* funcName, double& numValue, std
 ** Runs given string in the context of the script file.
 **
 */
-void LuaScript::RunString(const char* str)
+void LuaScript::RunString(const std::wstring& str)
 {
-	lua_State* L = LuaManager::GetState();
+	lua_State* L = GetState();
 
 	if (IsInitialized())
 	{
+		const std::string narrowStr = m_Unicode ?
+			StringUtil::NarrowUTF8(str) : StringUtil::Narrow(str);
+
 		// Load the string as a Lua chunk
-		if (luaL_loadstring(L, str))
+		if (luaL_loadstring(L, narrowStr.c_str()))
 		{
-			LuaManager::ReportErrors(L, m_File);
+			LuaManager::ReportErrors(m_File);
 		}
 
 		// Push our table onto the stack
@@ -243,7 +255,7 @@ void LuaScript::RunString(const char* str)
 
 		if (lua_pcall(L, 0, 0, 0))
 		{
-			LuaManager::ReportErrors(L, m_File);
+			LuaManager::ReportErrors(m_File);
 		}
 	}
 }

Library/lua/LuaScript.h

@@ -31,18 +31,21 @@ public:
 	void Uninitialize();
 	bool IsInitialized() { return m_Ref != LUA_NOREF; }
 
-	int GetRef() { return m_Ref; }
 	const std::wstring& GetFile() { return m_File; }
+	int GetRef() { return m_Ref; }
+	bool IsUnicode() const { return m_Unicode; }
+
+	lua_State* GetState() { return LuaManager::GetState(m_Unicode); }
 
 	bool IsFunction(const char* funcName);
 	void RunFunction(const char* funcName);
 	int RunFunctionWithReturn(const char* funcName, double& numValue, std::wstring& strValue);
-	void RunString(const char* str);
+	void RunString(const std::wstring& str);
 
 protected:
-	int m_Ref;
-
 	std::wstring m_File;
+	int m_Ref;
+	bool m_Unicode;
 };
 
 #endif

Library/lua/glue/LuaMeasure.cpp

@@ -29,7 +29,7 @@
 static int GetName(lua_State* L)
 {
 	DECLARE_SELF(L)
-	LuaManager::PushWide(L, self->GetName());
+	LuaManager::PushWide(self->GetName());
 
 	return 1;
 }
@@ -40,10 +40,10 @@ static int GetOption(lua_State* L)
 	MeterWindow* meterWindow = self->GetMeterWindow();
 	ConfigParser& parser = meterWindow->GetParser();
 
-	std::wstring strTmp = LuaManager::ToWide(L, 2);
+	std::wstring strTmp = LuaManager::ToWide(2);
-	strTmp = parser.ReadString(self->GetName(), strTmp.c_str(), LuaManager::ToWide(L, 3).c_str());
+	strTmp = parser.ReadString(self->GetName(), strTmp.c_str(), LuaManager::ToWide(3).c_str());
 
-	LuaManager::PushWide(L, strTmp);
+	LuaManager::PushWide(strTmp);
 	return 1;
 }
 
@@ -53,7 +53,7 @@ static int GetNumberOption(lua_State* L)
 	MeterWindow* meterWindow = self->GetMeterWindow();
 	ConfigParser& parser = meterWindow->GetParser();
 
-	std::wstring strTmp = LuaManager::ToWide(L, 2);
+	std::wstring strTmp = LuaManager::ToWide(2);
 	double value = parser.ReadFloat(self->GetName(), strTmp.c_str(), lua_tonumber(L, 3));
 
 	lua_pushnumber(L, value);
@@ -127,7 +127,7 @@ static int GetStringValue(lua_State* L)
 	bool percentual = lua_toboolean(L, 5);
 
 	const WCHAR* val = self->GetStringOrFormattedValue(autoScale, scale, decimals, percentual);
-	LuaManager::PushWide(L, val);
+	LuaManager::PushWide(val);
 
 	return 1;
 }

Library/lua/glue/LuaMeter.cpp

@@ -29,7 +29,7 @@
 static int GetName(lua_State* L)
 {
 	DECLARE_SELF(L)
-	LuaManager::PushWide(L, self->GetName());
+	LuaManager::PushWide(self->GetName());
 
 	return 1;
 }
@@ -40,10 +40,10 @@ static int GetOption(lua_State* L)
 	MeterWindow* meterWindow = self->GetMeterWindow();
 	ConfigParser& parser = meterWindow->GetParser();
 
-	std::wstring strTmp = LuaManager::ToWide(L, 2);
+	std::wstring strTmp = LuaManager::ToWide(2);
 	strTmp = parser.ReadString(self->GetName(), strTmp.c_str(), L"");
 
-	LuaManager::PushWide(L, strTmp);
+	LuaManager::PushWide(strTmp);
 	return 1;
 }
 
@@ -140,7 +140,7 @@ static int SetText(lua_State* L)
 	if (self->GetTypeID() == TypeID<MeterString>())
 	{
 		MeterString* string = (MeterString*)self;
-		std::wstring str = LuaManager::ToWide(L, 2);
+		std::wstring str = LuaManager::ToWide(2);
 		string->SetText(str.c_str());
 	}
 

Library/lua/glue/LuaMeterWindow.cpp

@@ -32,7 +32,7 @@ static int Bang(lua_State* L)
 	DECLARE_SELF(L)
 	ConfigParser& parser = self->GetParser();
 
-	std::wstring bang = LuaManager::ToWide(L, 2);
+	std::wstring bang = LuaManager::ToWide(2);
 
 	int top = lua_gettop(L);
 	if (top == 2)	// 1 argument
@@ -49,7 +49,7 @@ static int Bang(lua_State* L)
 			std::vector<std::wstring> args;
 			for (int i = 3; i <= top; ++i)
 			{
-				std::wstring tmpSz = LuaManager::ToWide(L, i);
+				std::wstring tmpSz = LuaManager::ToWide(i);
 				parser.ReplaceVariables(tmpSz);
 				args.push_back(tmpSz);
 			}
@@ -64,7 +64,7 @@ static int Bang(lua_State* L)
 static int GetMeter(lua_State* L)
 {
 	DECLARE_SELF(L)
-	const std::wstring meterName = LuaManager::ToWide(L, 2);
+	const std::wstring meterName = LuaManager::ToWide(2);
 
 	Meter* meter = self->GetMeter(meterName);
 	if (meter)
@@ -84,7 +84,7 @@ static int GetMeter(lua_State* L)
 static int GetMeasure(lua_State* L)
 {
 	DECLARE_SELF(L)
-	const std::wstring measureName = LuaManager::ToWide(L, 2);
+	const std::wstring measureName = LuaManager::ToWide(2);
 
 	Measure* measure = self->GetMeasure(measureName);
 	if (measure)
@@ -104,12 +104,12 @@ static int GetMeasure(lua_State* L)
 static int GetVariable(lua_State* L)
 {
 	DECLARE_SELF(L)
-	std::wstring strTmp = LuaManager::ToWide(L, 2);
+	std::wstring strTmp = LuaManager::ToWide(2);
 
 	const std::wstring* value = self->GetParser().GetVariable(strTmp);
 	if (value)
 	{
-		LuaManager::PushWide(L, *value);
+		LuaManager::PushWide(*value);
 	}
 	else
 	{
@@ -122,11 +122,11 @@ static int GetVariable(lua_State* L)
 static int ReplaceVariables(lua_State* L)
 {
 	DECLARE_SELF(L)
-	std::wstring strTmp = LuaManager::ToWide(L, 2);
+	std::wstring strTmp = LuaManager::ToWide(2);
 
 	self->GetParser().ReplaceVariables(strTmp);
 	self->GetParser().ReplaceMeasures(strTmp);
-	LuaManager::PushWide(L, strTmp);
+	LuaManager::PushWide(strTmp);
 
 	return 1;
 }
@@ -134,7 +134,7 @@ static int ReplaceVariables(lua_State* L)
 static int ParseFormula(lua_State* L)
 {
 	DECLARE_SELF(L)
-	std::wstring strTmp = LuaManager::ToWide(L, 2);
+	std::wstring strTmp = LuaManager::ToWide(2);
 
 	double result;
 	if (!self->GetParser().ParseFormula(strTmp, &result))
@@ -202,9 +202,9 @@ static int GetY(lua_State* L)
 static int MakePathAbsolute(lua_State* L)
 {
 	DECLARE_SELF(L)
-	std::wstring path = LuaManager::ToWide(L, 2);
+	std::wstring path = LuaManager::ToWide(2);
 	self->MakePathAbsolute(path);
-	LuaManager::PushWide(L, path);
+	LuaManager::PushWide(path);
 
 	return 1;
 }