From 6bdcaa0bfae2f2feecba3e006f32581da144721f Mon Sep 17 00:00:00 2001
From: gizmo98 <atz.SE@gmx.de>
Date: Wed, 26 Feb 2020 19:18:50 +0100
Subject: [PATCH] Add RDRAM dithering: Dither image if RGBA16 is written to
 RDRAM

ColorbufferToRDRAM RGBA to RGBA16 conversion produces color banding.
Implement dithering according to n64 documentation to hide color
banding.
---
 src/BufferCopy/ColorBufferToRDRAM.cpp | 44 ++++++++++++++++++++++++---
 src/BufferCopy/ColorBufferToRDRAM.h   |  6 ++--
 src/BufferCopy/DepthBufferToRDRAM.cpp |  2 +-
 src/BufferCopy/DepthBufferToRDRAM.h   |  2 +-
 src/BufferCopy/WriteToRDRAM.h         |  6 ++--
 5 files changed, 47 insertions(+), 13 deletions(-)

diff --git a/src/BufferCopy/ColorBufferToRDRAM.cpp b/src/BufferCopy/ColorBufferToRDRAM.cpp
index 5345c4e9..de71f87e 100644
--- a/src/BufferCopy/ColorBufferToRDRAM.cpp
+++ b/src/BufferCopy/ColorBufferToRDRAM.cpp
@@ -234,17 +234,51 @@ bool ColorBufferToRDRAM::_prepareCopy(u32& _startAddress)
 	return true;
 }
 
-u8 ColorBufferToRDRAM::_RGBAtoR8(u8 _c) {
+u8 ColorBufferToRDRAM::_RGBAtoR8(u8 _c, u32 x, u32 y) {
 	return _c;
 }
 
-u16 ColorBufferToRDRAM::_RGBAtoRGBA16(u32 _c) {
-	RGBA c;
+u16 ColorBufferToRDRAM::_RGBAtoRGBA16(u32 _c, u32 x, u32 y) {
+	// Precalculated 4x4 bayer matrix values for 5Bit
+	static const s32 thresholdMapBayer[4][4] = {
+		{ -4, 2, -3, 4 },
+		{ 0, -2, 2, -1 },
+		{ -3, 3, -4, 3 },
+		{ 1, -1, 1, -2 }
+	};
+
+	// Precalculated 4x4 magic square matrix values for 5Bit
+	static const s32 thresholdMapMagicSquare[4][4] = {
+		{ -4, 2, 2, -1 },
+		{ 3, -2, -3, 1 },
+		{ -3, 0, 4, -2 },
+		{ 3, -1, -4, 1 }
+	};
+
+	union RGBA c;
 	c.raw = _c;
-	return ((c.r >> 3) << 11) | ((c.g >> 3) << 6) | ((c.b >> 3) << 1) | (c.a == 0 ? 0 : 1);
+	
+	if(gDP.otherMode.colorDither <= 1) {
+		s32 threshold = 0;
+		
+		switch(gDP.otherMode.colorDither){
+			case G_CD_BAYER:
+				threshold = thresholdMapBayer[x & 3][y & 3];
+				break;
+			case G_CD_MAGICSQ:
+				threshold = thresholdMapMagicSquare[x & 3][y & 3];
+				break;
+		}
+		
+		c.r = (u8)std::max(std::min((s32)c.r + threshold, 255), 0);
+		c.g = (u8)std::max(std::min((s32)c.g + threshold, 255), 0);
+		c.b = (u8)std::max(std::min((s32)c.b + threshold, 255), 0);
+	}
+		
+	return ((c.r >> 3) << 11) | ((c.g >> 3) << 6) | ((c.b >> 3) << 1) | (c.a == 0 ? 0 : 1);	
 }
 
-u32 ColorBufferToRDRAM::_RGBAtoRGBA32(u32 _c) {
+u32 ColorBufferToRDRAM::_RGBAtoRGBA32(u32 _c, u32 x, u32 y) {
 	RGBA c;
 	c.raw = _c;
 	return (c.r << 24) | (c.g << 16) | (c.b << 8) | c.a;
diff --git a/src/BufferCopy/ColorBufferToRDRAM.h b/src/BufferCopy/ColorBufferToRDRAM.h
index 3d8fc726..3c20fd26 100644
--- a/src/BufferCopy/ColorBufferToRDRAM.h
+++ b/src/BufferCopy/ColorBufferToRDRAM.h
@@ -49,9 +49,9 @@ private:
 	u32 _getRealWidth(u32 _viWidth);
 
 	// Convert pixel from video memory to N64 buffer format.
-	static u8 _RGBAtoR8(u8 _c);
-	static u16 _RGBAtoRGBA16(u32 _c);
-	static u32 _RGBAtoRGBA32(u32 _c);
+	static u8 _RGBAtoR8(u8 _c, u32 x, u32 y);
+	static u16 _RGBAtoRGBA16(u32 _c, u32 x, u32 y);
+	static u32 _RGBAtoRGBA32(u32 _c, u32 x, u32 y);
 
 	graphics::ObjectHandle m_FBO;
 	FrameBuffer * m_pCurFrameBuffer;
diff --git a/src/BufferCopy/DepthBufferToRDRAM.cpp b/src/BufferCopy/DepthBufferToRDRAM.cpp
index f1f77983..9defc6e0 100644
--- a/src/BufferCopy/DepthBufferToRDRAM.cpp
+++ b/src/BufferCopy/DepthBufferToRDRAM.cpp
@@ -205,7 +205,7 @@ bool DepthBufferToRDRAM::_prepareCopy(u32& _startAddress, bool _copyChunk)
 	return true;
 }
 
-u16 DepthBufferToRDRAM::_FloatToUInt16(f32 _z)
+u16 DepthBufferToRDRAM::_FloatToUInt16(f32 _z, u32 x, u32 y)
 {
 	static const u16 * const zLUT = depthBufferList().getZLUT();
 	u32 idx = 0x3FFFF;
diff --git a/src/BufferCopy/DepthBufferToRDRAM.h b/src/BufferCopy/DepthBufferToRDRAM.h
index 71841113..f25681fe 100644
--- a/src/BufferCopy/DepthBufferToRDRAM.h
+++ b/src/BufferCopy/DepthBufferToRDRAM.h
@@ -30,7 +30,7 @@ private:
 	bool _copy(u32 _startAddress, u32 _endAddress);
 
 	// Convert pixel from video memory to N64 depth buffer format.
-	static u16 _FloatToUInt16(f32 _z);
+	static u16 _FloatToUInt16(f32 _z, u32 x, u32 y);
 
 	graphics::ObjectHandle m_FBO;
 	std::unique_ptr<graphics::PixelReadBuffer> m_pbuf;
diff --git a/src/BufferCopy/WriteToRDRAM.h b/src/BufferCopy/WriteToRDRAM.h
index 477eee87..8e9d2487 100644
--- a/src/BufferCopy/WriteToRDRAM.h
+++ b/src/BufferCopy/WriteToRDRAM.h
@@ -5,7 +5,7 @@
 #include "../Types.h"
 
 template <typename TSrc, typename TDst>
-void writeToRdram(TSrc* _src, TDst* _dst, TDst(*converter)(TSrc _c), TSrc _testValue, u32 _xor, u32 _width, u32 _height, u32 _numPixels, u32 _startAddress, u32 _bufferAddress, u32 _bufferSize)
+void writeToRdram(TSrc* _src, TDst* _dst, TDst(*converter)(TSrc _c, u32 x, u32 y), TSrc _testValue, u32 _xor, u32 _width, u32 _height, u32 _numPixels, u32 _startAddress, u32 _bufferAddress, u32 _bufferSize)
 {
 	u32 chunkStart = ((_startAddress - _bufferAddress) >> (_bufferSize - 1)) % _width;
 	if (chunkStart % 2 != 0) {
@@ -21,7 +21,7 @@ void writeToRdram(TSrc* _src, TDst* _dst, TDst(*converter)(TSrc _c), TSrc _testV
 		for (u32 x = chunkStart; x < _width; ++x) {
 			c = _src[x];
 			if (c != _testValue)
-				_dst[numStored ^ _xor] = converter(c);
+				_dst[numStored ^ _xor] = converter(c, x, y);
 			++numStored;
 		}
 		++y;
@@ -33,7 +33,7 @@ void writeToRdram(TSrc* _src, TDst* _dst, TDst(*converter)(TSrc _c), TSrc _testV
 		for (u32 x = 0; x < _width && numStored < _numPixels; ++x) {
 			c = _src[x + y *_width];
 			if (c != _testValue)
-				_dst[(x + dsty*_width) ^ _xor] = converter(c);
+				_dst[(x + dsty*_width) ^ _xor] = converter(c, x, y);
 			++numStored;
 		}
 		++dsty;