Implement alternative methods of combiners shaders load.

Combiners muxes saved in separate text file.
When shader cache becames invalid, list of shaders muxes used to re-create combiners.

src/Combiner.cpp

@@ -188,18 +188,16 @@ void SimplifyCycle( CombineCycle *cc, CombinerStage *stage )
 	}
 }
 
-//ShaderCombiner * CombinerInfo::_compile(u64 mux) const
-CombinerProgram * CombinerInfo::_compile(u64 mux) const
+graphics::CombinerProgram * Combiner_Compile(CombinerKey key)
 {
 	gDPCombine combine;
 
-	combine.mux = mux;
-
-	int numCycles;
+	combine.mux = key.getMux();
 
 	Combiner color, alpha;
 
-	numCycles = gDP.otherMode.cycleType + 1;
+	const u32 cycleType = key.getCycleType();
+	const u32 numCycles = cycleType + 1;
 	color.numStages = numCycles;
 	alpha.numStages = numCycles;
 
@@ -207,7 +205,7 @@ CombinerProgram * CombinerInfo::_compile(u64 mux) const
 	CombineCycle ac[2];
 
 	// Simplify each RDP combiner cycle into a combiner stage
-	if (gDP.otherMode.cycleType == G_CYC_1CYCLE) {
+	if (cycleType == G_CYC_1CYCLE) {
 		// 1 cycle mode uses combiner equations from 2nd cycle
 		u32 colorMux[4] = { saRGBExpanded[combine.saRGB1], sbRGBExpanded[combine.sbRGB1],
 							mRGBExpanded[combine.mRGB1], aRGBExpanded[combine.aRGB1] };
@@ -267,7 +265,7 @@ CombinerProgram * CombinerInfo::_compile(u64 mux) const
 		}
 	}
 
-	return gfxContext.createCombinerProgram(color, alpha, CombinerKey(mux));
+	return gfxContext.createCombinerProgram(color, alpha, key);
 }
 
 void CombinerInfo::update()
@@ -295,7 +293,7 @@ void CombinerInfo::setCombine(u64 _mux )
 	if (iter != m_combiners.end()) {
 		m_pCurrent = iter->second;
 	} else {
-		m_pCurrent = _compile(_mux);
+		m_pCurrent = Combiner_Compile(key);
 		m_pCurrent->update(true);
 		m_combiners[m_pCurrent->getKey()] = m_pCurrent;
 	}

src/Combiner.h

@@ -150,8 +150,6 @@ private:
 
 	void _saveShadersStorage() const;
 	bool _loadShadersStorage();
-	u32 _getConfigOptionsBitSet() const;
-	graphics::CombinerProgram * _compile(u64 mux) const;
 
 	bool m_bChanged;
 	bool m_bShaderCacheSupported;
@@ -173,6 +171,7 @@ graphics::CombinerProgram * currentCombiner() {
 
 void Combiner_Init();
 void Combiner_Destroy();
+graphics::CombinerProgram * Combiner_Compile(CombinerKey key);
 
 #endif
 

src/CombinerKey.cpp

@@ -4,9 +4,11 @@
 
 /*---------------CombinerKey-------------*/
 
-CombinerKey::CombinerKey(u64 _mux)
+CombinerKey::CombinerKey(u64 _mux, bool _setModeBits)
 {
 	m_key.mux = _mux;
+	if (!_setModeBits)
+		return;
 
 	// High byte of muxs0 is zero. We can use it for addtional combiner flags:
 	// [0 - 0] polygon type: 0 - triangle, 1 - rect

src/CombinerKey.h

@@ -7,7 +7,7 @@ public:
 	CombinerKey() {
 		m_key.mux = 0;
 	}
-	explicit CombinerKey(u64 _mux);
+	explicit CombinerKey(u64 _mux, bool _setModeBits = true);
 	CombinerKey(const CombinerKey & _other);
 
 	void operator=(u64 _mux);

src/Graphics/OpenGLContext/GLSL/glsl_ShaderStorage.cpp

@@ -1,6 +1,8 @@
 #include <fstream>
 #include <assert.h>
 #include <cstdlib>
+#include <algorithm>
+#include <iomanip>
 
 #include <Graphics/CombinerProgram.h>
 #include <Graphics/OpenGLContext/opengl_Utils.h>
@@ -8,6 +10,7 @@
 #include <Log.h>
 #include <RSP.h>
 #include <PluginAPI.h>
+#include <Combiner.h>
 #include <osal_files.h>
 #include "glsl_Utils.h"
 #include "glsl_ShaderStorage.h"
@@ -19,7 +22,7 @@ using namespace glsl;
 #define SHADER_STORAGE_FOLDER_NAME L"shaders"
 
 static
-void getStorageFileName(const opengl::GLInfo & _glinfo, wchar_t * _fileName)
+void getStorageFileName(const opengl::GLInfo & _glinfo, wchar_t * _shadersFileName, const wchar_t * _fileExtension)
 {
 	wchar_t strCacheFolderPath[PLUGIN_PATH_SIZE];
 	api().GetUserCachePath(strCacheFolderPath);
@@ -39,7 +42,7 @@ void getStorageFileName(const opengl::GLInfo & _glinfo, wchar_t * _fileName)
 		strOpenGLType = L"OpenGL";
 	}
 
-	swprintf(_fileName, PLUGIN_PATH_SIZE, L"%ls/GLideN64.%08lx.%ls.shaders", pPath, std::hash<std::string>()(RSP.romname), strOpenGLType.c_str());
+	swprintf(_shadersFileName, PLUGIN_PATH_SIZE, L"%ls/GLideN64.%08lx.%ls.%ls", pPath, std::hash<std::string>()(RSP.romname), strOpenGLType.c_str(), _fileExtension);
 }
 
 static
@@ -66,33 +69,38 @@ shaders in binary form
 */
 bool ShaderStorage::saveShadersStorage(const graphics::Combiners & _combiners) const
 {
-	wchar_t fileName[PLUGIN_PATH_SIZE];
-	getStorageFileName(m_glinfo, fileName);
+	wchar_t shadersFileName[PLUGIN_PATH_SIZE];
+	wchar_t keysFileName[PLUGIN_PATH_SIZE];
+	getStorageFileName(m_glinfo, shadersFileName, L"shaders");
+	getStorageFileName(m_glinfo, keysFileName, L"keys");
 
 #if defined(OS_WINDOWS) && !defined(MINGW)
-	std::ofstream fout(fileName, std::ofstream::binary | std::ofstream::trunc);
+	std::ofstream shadersOut(shadersFileName, std::ofstream::binary | std::ofstream::trunc);
+	std::ofstream keysOut(keysFileName, std::ofstream::trunc);
 #else
-	char fileName_c[PATH_MAX];
-	wcstombs(fileName_c, fileName, PATH_MAX);
-	std::ofstream fout(fileName_c, std::ofstream::binary | std::ofstream::trunc);
+	char filename_c[PATH_MAX];
+	wcstombs(filename_c, shadersFileName, PATH_MAX);
+	std::ofstream shadersOut(filename_c, std::ofstream::binary | std::ofstream::trunc);
+	wcstombs(filename_c, keysFileName, PATH_MAX);
+	std::ofstream keysOut(filename_c, std::ofstream::trunc);
 #endif
-	if (!fout)
+	if (!shadersOut || !keysOut)
 		return false;
 
-	fout.write((char*)&m_formatVersion, sizeof(m_formatVersion));
+	shadersOut.write((char*)&m_formatVersion, sizeof(m_formatVersion));
 
 	const u32 configOptionsBitSet = _getConfigOptionsBitSet();
-	fout.write((char*)&configOptionsBitSet, sizeof(configOptionsBitSet));
+	shadersOut.write((char*)&configOptionsBitSet, sizeof(configOptionsBitSet));
 
 	const char * strRenderer = reinterpret_cast<const char *>(glGetString(GL_RENDERER));
 	u32 len = strlen(strRenderer);
-	fout.write((char*)&len, sizeof(len));
-	fout.write(strRenderer, len);
+	shadersOut.write((char*)&len, sizeof(len));
+	shadersOut.write(strRenderer, len);
 
 	const char * strGLVersion = reinterpret_cast<const char *>(glGetString(GL_VERSION));
 	len = strlen(strGLVersion);
-	fout.write((char*)&len, sizeof(len));
-	fout.write(strGLVersion, len);
+	shadersOut.write((char*)&len, sizeof(len));
+	shadersOut.write(strGLVersion, len);
 
 	len = _combiners.size();
 
@@ -103,12 +111,15 @@ bool ShaderStorage::saveShadersStorage(const graphics::Combiners & _combiners) c
 #else
 	u32 totalWritten = 0;
 	std::vector<char> allShaderData;
+	std::vector<u64> keysData;
+	keysData.reserve(len);
 
 	for (auto cur = _combiners.begin(); cur != _combiners.end(); ++cur)
 	{
 		std::vector<char> data;
 		if (cur->second->getBinaryForm(data))
 		{
+			keysData.push_back(cur->first.getMux());
 			allShaderData.insert(allShaderData.end(), data.begin(), data.end());
 			++totalWritten;
 		}
@@ -119,12 +130,20 @@ bool ShaderStorage::saveShadersStorage(const graphics::Combiners & _combiners) c
 		}
 	}
 
-	fout.write((char*)&totalWritten, sizeof(totalWritten));
-	fout.write(allShaderData.data(), allShaderData.size());
+	shadersOut.write((char*)&totalWritten, sizeof(totalWritten));
+	shadersOut.write(allShaderData.data(), allShaderData.size());
+
+	std::sort(keysData.begin(), keysData.end());
+	keysOut << "0x" << std::hex << std::setfill('0') << std::setw(8) << m_keysFormatVersion << "\n";
+	for (u64 key : keysData)
+		keysOut << "0x" << std::hex << std::setfill('0') << std::setw(16) << key << "\n";
+
 #endif
 
-	fout.flush();
-	fout.close();
+	shadersOut.flush();
+	shadersOut.close();
+	keysOut.flush();
+	keysOut.close();
 	return true;
 }
 
@@ -159,32 +178,67 @@ CombinerProgramImpl * _readCominerProgramFromStream(std::istream & _is,
 	return new CombinerProgramImpl(cmbKey, program, _useProgram, cmbInputs, std::move(uniforms));
 }
 
+bool ShaderStorage::_loadFromCombinerKeys(graphics::Combiners & _combiners)
+{
+	wchar_t keysFileName[PLUGIN_PATH_SIZE];
+	getStorageFileName(m_glinfo, keysFileName, L"keys");
+#if defined(OS_WINDOWS) && !defined(MINGW)
+	std::ifstream fin(keysFileName);
+#else
+	char fileName_c[PATH_MAX];
+	wcstombs(fileName_c, keysFileName, PATH_MAX);
+	std::ifstream fin(fileName_c);
+#endif
+	if (!fin)
+		return false;
+
+	u32 version;
+	fin >> std::hex >> version;
+	if (version != m_keysFormatVersion)
+		return false;
+
+	u64 key;
+	while (!fin.eof()) {
+		fin >> std::hex >> key;
+		graphics::CombinerProgram * pCombiner = Combiner_Compile(CombinerKey(key, false));
+		pCombiner->update(true);
+		_combiners[pCombiner->getKey()] = pCombiner;
+	}
+	fin.close();
+
+	if (opengl::Utils::isGLError())
+		return false;
+
+	return saveShadersStorage(_combiners);
+}
+
 bool ShaderStorage::loadShadersStorage(graphics::Combiners & _combiners)
 {
-	wchar_t fileName[PLUGIN_PATH_SIZE];
-	getStorageFileName(m_glinfo, fileName);
+	wchar_t shadersFileName[PLUGIN_PATH_SIZE];
+	getStorageFileName(m_glinfo, shadersFileName, L"shaders");
 	const u32 configOptionsBitSet = _getConfigOptionsBitSet();
 
 #if defined(OS_WINDOWS) && !defined(MINGW)
-	std::ifstream fin(fileName, std::ofstream::binary);
+	std::ifstream fin(shadersFileName, std::ofstream::binary);
 #else
 	char fileName_c[PATH_MAX];
 	wcstombs(fileName_c, fileName, PATH_MAX);
 	std::ifstream fin(fileName_c, std::ofstream::binary);
 #endif
+
 	if (!fin)
-		return false;
+		return _loadFromCombinerKeys(_combiners);
 
 	try {
 		u32 version;
 		fin.read((char*)&version, sizeof(version));
 		if (version != m_formatVersion)
-			return false;
+			return _loadFromCombinerKeys(_combiners);
 
 		u32 optionsSet;
 		fin.read((char*)&optionsSet, sizeof(optionsSet));
 		if (optionsSet != configOptionsBitSet)
-			return false;
+			return _loadFromCombinerKeys(_combiners);
 
 		const char * strRenderer = reinterpret_cast<const char *>(glGetString(GL_RENDERER));
 		u32 len;
@@ -192,14 +246,14 @@ bool ShaderStorage::loadShadersStorage(graphics::Combiners & _combiners)
 		std::vector<char> strBuf(len);
 		fin.read(strBuf.data(), len);
 		if (strncmp(strRenderer, strBuf.data(), len) != 0)
-			return false;
+			return _loadFromCombinerKeys(_combiners);
 
 		const char * strGLVersion = reinterpret_cast<const char *>(glGetString(GL_VERSION));
 		fin.read((char*)&len, sizeof(len));
 		strBuf.resize(len);
 		fin.read(strBuf.data(), len);
 		if (strncmp(strGLVersion, strBuf.data(), len) != 0)
-			return false;
+			return _loadFromCombinerKeys(_combiners);
 
 		CombinerProgramUniformFactory uniformFactory(m_glinfo);
 

src/Graphics/OpenGLContext/GLSL/glsl_ShaderStorage.h

@@ -17,7 +17,10 @@ namespace glsl {
 		bool loadShadersStorage(graphics::Combiners & _combiners);
 
 	private:
+		bool _loadFromCombinerKeys(graphics::Combiners & _combiners);
+
 		const u32 m_formatVersion = 0x14U;
+		const u32 m_keysFormatVersion = 0x01;
 		const opengl::GLInfo & m_glinfo;
 		opengl::CachedUseProgram * m_useProgram;
 	};