Outline Shader - fixed

The required infos were squirreled away at the gloopy end of the internets ...

shaders/common/postFx/OutlineShaderP.hlsl

#include "postFx.hlsl"
#include "shadergen:/autogenConditioners.h"

float4 main( PFXVertToPix IN, 
             uniform sampler2D edgeBuffer :register(S0), uniform sampler2D backBuffer : register(S1) ) : COLOR0
{
   float4 e = float4( tex2D( edgeBuffer, IN.uv0 ).rrr, 1.0 );
   e.r = e.g = e.b = 1.0 - e.r;
   return tex2D(backBuffer, IN.uv0) * e;
}

shaders/common/postFx/OutlineDetectP.hlsl

#include "postFx.hlsl"
#include "shadergen:/autogenConditioners.h"

// GPU Gems 3, pg 443-444
float GetEdgeWeight(float2 uv0, in sampler2D prepassBuffer, in float2 targetSize)
{
   float2 offsets[9] = {
      float2( 0.0,  0.0),
      float2(-1.0, -1.0),
      float2( 0.0, -1.0),
      float2( 1.0, -1.0),
      float2( 1.0,  0.0),
      float2( 1.0,  1.0),
      float2( 0.0,  1.0),
      float2(-1.0,  1.0),
      float2(-1.0,  0.0),
   };
   
   
   float2 PixelSize = 1.0 / targetSize;
   
   float Depth[9];
   float3 Normal[9];
   
   for(int i = 0; i < 9; i++)
   {
      float2 uv = uv0 + offsets[i] * PixelSize;
      float4 gbSample = prepassUncondition( tex2D(prepassBuffer, uv) );
      Depth[i] = gbSample.a;
      Normal[i] = gbSample.rgb;
   }
   
   float4 Deltas1 = float4(Depth[1], Depth[2], Depth[3], Depth[4]);
   float4 Deltas2 = float4(Depth[5], Depth[6], Depth[7], Depth[8]);
   
   Deltas1 = abs(Deltas1 - Depth[0]);
   Deltas2 = abs(Depth[0] - Deltas2);
   
   float4 maxDeltas = max(Deltas1, Deltas2);
   float4 minDeltas = max(min(Deltas1, Deltas2), 0.00001);
   
   float4 depthResults = step(minDeltas * 25.0, maxDeltas);
   
   Deltas1.x = dot(Normal[1], Normal[0]);
   Deltas1.y = dot(Normal[2], Normal[0]);
   Deltas1.z = dot(Normal[3], Normal[0]);
   Deltas1.w = dot(Normal[4], Normal[0]);
   
   Deltas2.x = dot(Normal[5], Normal[0]);
   Deltas2.y = dot(Normal[6], Normal[0]);
   Deltas2.z = dot(Normal[7], Normal[0]);
   Deltas2.w = dot(Normal[8], Normal[0]);
   
   Deltas1 = abs(Deltas1 - Deltas2);
   
   float4 normalResults = step(0.4, Deltas1);
   
   normalResults = max(normalResults, depthResults);
   
   return dot(normalResults, float4(1.0, 1.0, 1.0, 1.0)) * 0.25;
}

float4 main( PFXVertToPix IN, 
             uniform sampler2D prepassBuffer :register(S0),
             uniform float2 targetSize : register(C0) ) : COLOR0
{
   return GetEdgeWeight(IN.uv0, prepassBuffer, targetSize );
}

core/scripts/client/postFx/edgeDetect.cs

singleton GFXStateBlockData( PFX_DefaultOutlineStateBlock )
{
   zDefined = true;
   zEnable = false;
   zWriteEnable = false;
      
   samplersDefined = true;
   samplerStates[0] = SamplerClampLinear;
};

singleton ShaderData( PFX_OutlineShader )
{   
   DXVertexShaderFile 	= "shaders/common/postFx/postFxV.hlsl";
   DXPixelShaderFile 	= "shaders/common/postFx/outlineShaderP.hlsl";
         
   samplerNames[0] = "$inputTex";
   
   pixVersion = 3.0;
};

singleton ShaderData( PFX_OutlineEdgeDetectShader )
{   
   DXVertexShaderFile 	= "shaders/common/postFx/postFxV.hlsl";
   DXPixelShaderFile 	= "shaders/common/postFx/edgeaa/edgeDetectP.hlsl";
 
   samplerNames[0] = "$inputTex";
   
   pixVersion = 3.0;
};

singleton PostEffect( OutlinerPostFX )
{
   renderTime = "PFXAfterDiffuse";
      
   shader = PFX_OutlineEdgeDetectShader;
   stateBlock = PFX_DefaultOutlineStateBlock;
   texture[0] = "#prepass";
   target = "$outTex";
   
   new PostEffect()
   {
      shader = PFX_OutlineShader;
      stateBlock = PFX_DefaultOutlineStateBlock;
      texture[0] = "$inTex"; 
      texture[1] = "$backBuffer";
      target = "$backBuffer";
   };
};

added to core/scripts/client/postFx.cs

//yorks in
singleton GFXStateBlockData( PFX_DefaultOutlineStateBlock )
{
   zDefined = true;
   zEnable = true;//false
   zWriteEnable = true;//false
      
   samplersDefined = true;
   samplerStates[0] = SamplerClampLinear;
   //samplerStates[1] = SamplerWrapPoint;
};

singleton ShaderData( PFX_OutlineShader )
{   
   DXVertexShaderFile 	= "shaders/common/postFx/postFxV.hlsl";
   DXPixelShaderFile 	= "shaders/common/postFx/outlineShaderP.hlsl";
      
   samplerNames[0] = "$inputTex";
   
   pixVersion = 3.0;
};

singleton ShaderData( PFX_OutlineEdgeDetectShader )
{   
   DXVertexShaderFile 	= "shaders/common/postFx/postFxV.hlsl";
   DXPixelShaderFile 	= "shaders/common/postFx/outlineDetectP.hlsl";
         
     
   samplerNames[0] = "$inputTex";
   
   pixVersion = 3.0;
};

singleton PostEffect( OutlineFX)
{
   renderTime = "PFXAfterDiffuse";
   //renderBin = "ObjTranslucentBin";      
   //renderPriority = 0.1;
      
   shader = PFX_OutlineEdgeDetectShader;
   stateBlock = PFX_DefaultOutlineStateBlock;
   texture[0] = "#prepass";
   target = "$outTex"; // "$backBuffer";
   
   new PostEffect()
   {
      shader = PFX_OutlineShader;
      stateBlock = PFX_DefaultOutlineStateBlock;
      texture[0] = "$inTex"; 
      texture[1] = "$backBuffer";
      target = "$backBuffer";
   };
};
//yorks

outlineFX.enable();