Watercolor: Add vignette option, improve color quantisation

Data Files/MWSE/mods/mer/joyOfPainting/interops/artStyle.lua

@@ -13,7 +13,6 @@ local shaders = {
     { id = "blackAndWhite", shaderId = "jop_blackwhite" },
     { id = "ink", shaderId = "jop_ink" },
     { id = "oil", shaderId = "jop_oil" },
-    { id = "vignette", shaderId = "jop_vignette" },
     { id = "watercolor", shaderId = "jop_watercolor" },
     { id = "window", shaderId = "jop_window" },
     { id = "detail", shaderId = "jop_kuwahara", defaultControls = {"brushSize"} },
@@ -36,15 +35,9 @@ local shaders = {
             "compositeBlacken",
         }
     },
-    {
-        id = "hatch",
-        shaderId = "jop_hatch",
-        defaultControls = { "hatchSize" }
-    },
-    {
-        id = "mottle",
-        shaderId = "jop_mottle",
-    }
+    { id = "hatch", shaderId = "jop_hatch", defaultControls = { "hatchSize" } },
+    { id = "mottle", shaderId = "jop_mottle" },
+    { id = "quantize", shaderId = "jop_quantize" },
 }
 
 ---@type JOP.ArtStyle.control[]
@@ -397,6 +390,17 @@ local controls = {
                 0.8, 0.2
             )
         end
+    },
+    {
+        id = "vignette",
+        uniform = "vignetteStrength",
+        shader = "jop_composite",
+        name = "Vignette",
+        sliderDefault = 0,
+        sliderMin = 0,
+        sliderMax = 1,
+        shaderMin = 0.0,
+        shaderMax = 1.0,
     }
 }
 
@@ -422,7 +426,7 @@ local artStyles = {
         paintType = "charcoal",
         maxDetailSkill = 30,
         minBrushSize = 3,
-        maxBrushSize = 15,
+        maxBrushSize = 12,
         helpText = [[
 Charcoal drawings work best with high contrast images against an empty background.
 
@@ -448,7 +452,7 @@ Use the fog setting to remove background elements and the threshold to adjust th
         paintType = "ink",
         maxDetailSkill = 40,
         minBrushSize = 2,
-        maxBrushSize = 15,
+        maxBrushSize = 12,
         helpText = [[
 Ink sketches are good for images with defined shapes.
 
@@ -464,10 +468,12 @@ Use the detail setting to adjust how dense the lines are, and the fog setting to
             "distort",
             "adjuster",
             "fogColor",
-            "composite"
+            "composite",
+            "quantize",
         },
         controls = {
             "watercolorLut",
+            "vignette",
             "brightness",
             "contrast",
             "canvasStrengthWatercolor",
@@ -480,7 +486,7 @@ Use the detail setting to adjust how dense the lines are, and the fog setting to
         --requiresEasel = true,
         maxDetailSkill = 50,
         minBrushSize = 6,
-        maxBrushSize = 15,
+        maxBrushSize = 12,
         helpText = [[
 Watercolor paintings have a limited color palette and thick brush strokes. They are good for making abstract and impressionist paintings.
 
@@ -511,7 +517,7 @@ Try replacing the background with the fog setting and changing the fog color to
         requiresEasel = true,
         maxDetailSkill = 60,
         minBrushSize = 3,
-        maxBrushSize = 15,
+        maxBrushSize = 12,
         helpText = [[
 Oil paintings require high skill before they start looking detailed.
 
@@ -542,7 +548,7 @@ Reduce contrast for a more matte look, or increase contrast to create more defin
         paintType = "pencil",
         maxDetailSkill = 55,
         minBrushSize = 2,
-        maxBrushSize = 15,
+        maxBrushSize = 12,
         helpText = [[
 The bright areas of the pencil drawing will be replaced with the background. Keep this in mind when preparing your scene, use the contrast/brightness settings to make sure any parts of the image you want to remain are below 50% brightness.
 ]]

Data Files/shaders/XEShaders/jop_charcoal.fx

@@ -7,22 +7,20 @@ texture lastpass;
 texture tex1 < string src="jop/pencil_tile.tga"; >;
 
 sampler sImage = sampler_state { texture=<lastshader>; minfilter = linear; magfilter = linear; mipfilter = linear; addressu=clamp; addressv = clamp;};
-sampler sScrollTex = sampler_state { texture=<tex1>; minfilter = linear; magfilter = linear; mipfilter = linear; addressu=wrap; addressv = wrap;};
+sampler sHatch = sampler_state { texture=<tex1>; minfilter = linear; magfilter = linear; mipfilter = linear; addressu=wrap; addressv = wrap;};
 sampler sLastpass = sampler_state { texture=<lastpass>; minfilter = linear; magfilter = linear; mipfilter = linear; addressu=clamp; addressv = clamp;};
 
 
 float4 main(float2 Tex : TEXCOORD0) : COLOR0 {
     float4 color = tex2D(sImage, Tex);
-
+    // Adjust the texture coordinates to match the pencil texture
     float2 adjustedTex = float2(Tex.x * rcpres.y / rcpres.x, Tex.y) *(1/pencil_scale);
-    float4 pencil = tex2D(sScrollTex, adjustedTex);
-
+    // Sample the pencil texture
+    float4 pencil = tex2D(sHatch, adjustedTex);
     // Color dodge effect approximation
     float4 result = color / (1.0 - pencil);
-
     // Clamping the result to avoid overflow
     result = clamp(result, 0.0, 1.0);
-
     return lerp(color, result, pencil_strength);
 }
 

Data Files/shaders/XEShaders/jop_composite.fx

@@ -1,15 +1,17 @@
 //When true, the overlay image will converted to black
 extern float doBlackenImage = false;
 //Higher values replace more of the overlay image with the canvas image
-extern float compositeStrength = 0.0;
+extern float compositeStrength = 0.2;
 //The aspect ratio of the canvas window
-extern float aspectRatio = 1.0;
+extern float aspectRatio = 1.3;
 //The size of the canvas window as a percentage of the screen
 extern float viewportSize = 0.8;
 //When true, the canvas window is rotated 90 degrees
 extern float isRotated;
 //The distance at which fog begins to obscure objects
 extern float fogDistance = 250;
+//If enabled, will use the alpha mask to create a vignette effect
+extern float vignetteStrength = 0;
 
 float maxDistance = 250-1;
 float2 rcpres;
@@ -20,18 +22,20 @@ texture lastshader;
 texture lastpass;
 texture depthframe;
 texture tex1  < string src="jop/composite_tex.dds"; >;
+texture tex2 < string src="jop/vignetteAlphaMask.tga"; >;
 sampler2D sLastShader = sampler_state { texture = <lastshader>; addressu = clamp; };
 sampler2D sLastPass = sampler_state { texture = <lastpass>; addressu = clamp; addressv = clamp; magfilter = point; minfilter = point; };
 sampler sDepthFrame = sampler_state { texture=<depthframe>; addressu = clamp; addressv = clamp; magfilter = point; minfilter = point; };
 sampler2D sComposite = sampler_state { texture=<tex1>; minfilter = linear; magfilter = linear; mipfilter = linear; addressu=wrap; addressv = wrap;};
+sampler2D sVignetteAlphaMask = sampler_state { texture =<tex2>; addressu = clamp; addressv = clamp; magfilter = linear; minfilter = linear; mipfilter = linear; };
 
 float readDepth(float2 tex)
 {
 	float depth = pow(tex2D(sDepthFrame, tex).r,1);
 	return depth;
 }
 
-float4 renderCanvas(float2 tex : TEXCOORD0) : COLOR0
+float4 renderCanvas(float2 tex, sampler2D image) : COLOR0
 {
     // Calculate the aspect ratio of the screen
     float screenRatio = screen_width / screen_height;
@@ -73,30 +77,29 @@ float4 renderCanvas(float2 tex : TEXCOORD0) : COLOR0
         mappedTex.x = (rotatedTex.x - newMin.x) / (newMax.x - newMin.x);
         mappedTex.y = (rotatedTex.y - newMin.y) / (newMax.y - newMin.y);
 
-        float4 canvas = tex2D(sComposite, mappedTex);
-        //Where the canvas has alpha, render as black
-        if (canvas.a < 0.5) {
-            return float4(0.01, 0.01, 0.01, 0);
-        }
+        float4 canvas = tex2D(image, mappedTex);
         return canvas; // Render pixel with mapped and rotated texture coordinates
     }
 }
 
+
 //This takes composites the sLastShader onto the result of sLastPass.
 //It renders the sLastShader transparent based on brightness and the compositeStrength.
 //At 0 compositeStrength, the sLastShader is invisible.
 //At 1 compositeStrength, the sLastShader is fully visible.
 float4 composite(float2 tex : TEXCOORD0) : COLOR0
 {
     float4 image = tex2D(sLastShader, tex);
-    float4 composite = tex2D(sLastPass, tex);
+    float4 composite = renderCanvas(tex, sComposite);
+    float4 alphaMask = renderCanvas(tex, sVignetteAlphaMask);
 
     // Calculate the brightness of the sLastShader
-    float brightness = dot(image.rgb, float3(0.299, 0.587, 0.114));
+    float brightness = max(max(image.r, image.g), image.b);
 
     // Calculate the final image based on compositeStrength
     float4 overlay = lerp(image, float4(0.01,0.01,0.01,image.a), doBlackenImage);
     float4 result = lerp(overlay, composite, saturate(brightness * compositeStrength));
+    result = lerp(result, composite, (1-alphaMask.a) * vignetteStrength);
 
     // Cull distant objects
     float depth = readDepth(tex);
@@ -116,6 +119,5 @@ float4 composite(float2 tex : TEXCOORD0) : COLOR0
 //priority adjusted to 100,000,000 above final because this REALLY can not be overwritten without breaking the mod
 technique T0 < string MGEinterface="MGE XE 0"; string category = "final"; int priorityAdjust = 500; >
 {
-	pass p0 { PixelShader = compile ps_3_0 renderCanvas(); }
     pass p1 { PixelShader = compile ps_3_0 composite(); }
 }

Data Files/shaders/XEShaders/jop_fog_bw.fx

@@ -12,7 +12,7 @@ sampler s1 = sampler_state { texture=<depthframe>; addressu = clamp; addressv =
 
 float readDepth(float2 tex)
 {
-	float depth = pow(tex2D(s1, tex).r,1);
+	float depth = tex2D(s1, tex);
 	return depth;
 }
 

Data Files/shaders/XEShaders/jop_mottle.fx

@@ -1,11 +1,11 @@
 extern float mottleStrength = 0.03;
 extern float mottleSize = 1.0;
-extern float sampleSpeed = 0.05;
-
+extern float sampleSpeed = 0.5;
+float2 rcpres;
 float time;
 
 texture lastshader;
-texture tex1 < string src="jop/waterbrush.dds"; >;
+texture tex1 < string src="jop/splash_watercolor.tga"; >;
 
 sampler sLastShader = sampler_state { texture = <lastshader>; addressu = mirror; addressv = mirror; magfilter = linear; minfilter = linear; };
 sampler sMottle = sampler_state { texture = <tex1>; addressu = wrap; addressv = wrap; magfilter = linear; minfilter = linear; };
@@ -15,44 +15,23 @@ sampler sMottle = sampler_state { texture = <tex1>; addressu = wrap; addressv =
 float3 applyMottlingEffect(float2 uv, float3 baseColor)
 {
     float3 newColor = baseColor; // Initialize new color
+    //newColor = float3(0,0,0);
+
+    //LumEffect: mottle is stronger at low luminosity
+    float luminosity = (newColor.r + newColor.g + newColor.b) / 3;
+    float lumEffect = 1 - luminosity;
 
-    //Darken
-    float2 randomUv1 = float2(uv.x + sin(time) * sampleSpeed, uv.y + cos(time) * sampleSpeed);
+    //Lighten based on luminosity of mottle color
+    float2 randomUv1 = float2(uv.x + sin(time * sampleSpeed) * 0.05, uv.y + cos(time* sampleSpeed) * 0.05);
     float3 mottleColor1 = tex2D(sMottle, randomUv1 / mottleSize); // Sample mottle texture
-    newColor = newColor + lerp(-mottleStrength, 0, mottleColor1.r);
 
-    //Lighten
-    float time2 = time * 0.5;
-    float2 randomUv2 = float2(uv.x + sin(time2) * sampleSpeed, uv.y + cos(time2) * sampleSpeed);
-    float3 mottleColor2 = tex2D(sMottle, randomUv2 / mottleSize); // Sample mottle texture
-    newColor = newColor + lerp(0, mottleStrength, mottleColor2.g);
+    //Squeeze the base color down by 10% and add the mottle color
+    newColor = newColor + mottleColor1 * mottleStrength * lumEffect;
 
     return newColor; // Blend base color with mottled color
 }
 
 
-float2 rotateUvByNormal(float2 uv, float3 normal)
-{
-    // Step 1: Calculate rotation axis and angle
-    float3 upVector = float3(0, 1, 0);
-    float3 rotationAxis = cross(upVector, normal);
-    float angle = acos(dot(normalize(upVector), normalize(normal)));
-
-    // Step 2: Create rotation matrix
-    float c = cos(angle);
-    float s = sin(angle);
-    float t = 1.0 - c;
-    float x = rotationAxis.x, y = rotationAxis.y, z = rotationAxis.z;
-    float3x3 rotationMatrix = float3x3(
-        t*x*x + c,    t*x*y - s*z,  t*x*z + s*y,
-        t*x*y + s*z,  t*y*y + c,    t*y*z - s*x,
-        t*x*z - s*y,  t*y*z + s*x,  t*z*z + c
-    );
-
-    // Step 3: Apply rotation to the hatch pattern
-    float2 rotatedHatchPosition = mul(rotationMatrix, uv);
-    return rotatedHatchPosition;
-}
 
 float4 main(float2 tex : TEXCOORD0) : COLOR0
 {

Data Files/shaders/XEShaders/jop_quantize.fx

@@ -0,0 +1,24 @@
+// Number of luminosity levels to quantize into
+extern int luminosityLevels = 16;
+
+texture lastshader;
+sampler2D sLastShader = sampler_state { texture = <lastshader>; addressu = clamp; };
+
+float4 main(float2 uv : TEXCOORD) : SV_Target {
+    // Sample the texture color
+    float4 texColor = tex2D(sLastShader, uv);
+    // Calculate the original luminosity
+    float luminosity = dot(texColor.rgb, float3(0.299, 0.587, 0.114));
+    // Quantize luminosity into a fixed number of levels
+    float quantizedLuminosity = floor(luminosity * luminosityLevels) / (luminosityLevels-1);
+    // Adjust the color's brightness to match the quantized luminosity
+    // Keeping the color's hue and saturation unchanged
+    float3 adjustedColor = texColor.rgb * (quantizedLuminosity / luminosity);
+    // Return the adjusted color with the original alpha
+    return float4(adjustedColor, texColor.a);
+}
+
+technique T0 < string MGEinterface="MGE XE 0"; string category = "final"; int priorityAdjust = 40; >
+{
+	pass p0 { PixelShader = compile ps_3_0 main(); }
+}

Data Files/shaders/XEShaders/jop_watercolor.fx

@@ -10,19 +10,18 @@ float2 rcpres;
 extern int selectedLut = 1;
 
 texture lastshader;
-texture depthframe;
 texture lastpass;
 
 
-texture tex1 < string src="jop/luts/saturated.tga"; >;
-texture tex2 < string src="jop/luts/neutral.tga"; >;
+texture tex1 < string src="jop/luts/neutral.tga"; >;
+texture tex2 < string src="jop/luts/saturated.tga"; >;
 texture tex3 < string src="jop/luts/desaturated.tga"; >;
 texture tex4 < string src="jop/luts/warm.tga"; >;
 texture tex5 < string src="jop/luts/cold.tga"; >;
-texture tex6 < string src="jop/luts/quantized_64.tga"; >;
-texture tex7 < string src="jop/luts/hueShifted.tga"; >;
-texture tex8 < string src="jop/luts/hueShift2.tga"; >;
-texture tex9 < string src="jop/luts/blackandwhite.tga"; >;
+texture tex6 < string src="jop/luts/hueShifted_1.tga"; >;
+texture tex7 < string src="jop/luts/hueShifted_2.tga"; >;
+texture tex8 < string src="jop/luts/hueShifted_3.tga"; >;
+texture tex9 < string src="jop/luts/sepia.tga"; >;
 texture tex10 < string src="jop/luts/radioactive.tga"; >;
 
 sampler sLastShader = sampler_state { texture=<lastshader>; addressu = clamp; addressv = clamp; magfilter = point; minfilter = point; };