Game Development Community

Custom shader: Writing normal map to Gbuffer

by Felix Willebrand Westin · in Torque 3D Professional · 05/31/2014 (4:47 pm) · 7 replies

I'm currently trying to write a custom skin/character shader with some success so far:

puu.sh/9aAzg.jpg

Currently it works well with the deferred lighting, however while trying to add normal mapping to it I've run into some issues.

From looking at the procedural shaders it seems to me that there are two sets of vertex/pixel shaders per material. One that takes care of the normals/depth and writes that to the Gbuffer, and one that does all the other stuff.

My question is, how does one go about doing this with a custom material? Is it even possible?
Right now the shader consists of just one vertex/pixel file respectively and I don't see a way of writing to the normal channel of the gbuffer, if that is indeed what needs to be done..

Here's the shader code if anyone is interested:

skinV.hlsl:
//*****************************************************************************
// West for the Wicked - Character skin shader
//*****************************************************************************

#include "shaders/common/hlslStructs.h"
#include "shaders/common/lighting.hlsl"
#include "shaders/common/torque.hlsl"

// Matrix to convert into view space
uniform float4x4	modelview;
// Matrix to go from object to cube map space
uniform float3x3	cubeTrans;
// Eye position relative to the cube map
uniform float3		cubeEyePos;
// Eye position relative to the object
uniform float3		eyePos;

// Output to pixel shader
struct VS_OUTPUT
{
	// Standard position not passed to the pixel shader
	float4 pos					: POSITION;
	// Texture coordinates
	float2 uv0					: TEXCOORD0;
	// Calculated reflection vector used by the pixel shader to perform a cube map lookup
	float3 reflectVec			: TEXCOORD1;
	// Calculated reflection scale that depends on the relationship between the surface normal and the eye position
	float reflectScale			: TEXCOORD2;
	// Pass the screen space position used for lighting calculations
	float4 screenspacePos		: TEXCOORD3;
	// Normals
	float3x3 outViewToTangent	: TEXCOORD4;
};

VS_OUTPUT main(VertexIn_PNTTTB IN)
{
	VS_OUTPUT OUT = (VS_OUTPUT)0;
	
	// Calculate the vertex position for the view
	OUT.pos = mul(modelview,IN.pos);
	
	// Pass along the texture coordinates
	OUT.uv0 = IN.uv0;
	
	// Calculate the reflection vector
	float3 cubeVertPos = mul(cubeTrans, IN.pos.xyz);
	float3 cubeNormal = normalize( mul( cubeTrans, normalize(IN.normal) ).xyz );
	float3 eyeToVert = cubeVertPos - cubeEyePos;
	OUT.reflectVec = reflect(eyeToVert, cubeNormal);
	
	// Fresnel power
	float power = 1;
	
	// Calculate the amount to scale the reflection by
	float3 eyeVec = normalize( eyePos.xyz - IN.pos.xyz );
	OUT.reflectScale = saturate( pow( abs( dot( eyeVec, IN.normal.xyz ) ), power ) );
	
	// Store the screen space position for RT lighting calculations
	OUT.screenspacePos = OUT.pos;
	
	// Return the output struct to the system
	return OUT;
}

skinP.hlsl:
//*****************************************************************************
// West for the Wicked - Character skin shader
//*****************************************************************************

#include "shadergen:/autogenConditioners.h"
#include "shaders/common/lighting.hlsl"
#include "shaders/common/torque.hlsl"

// Diffuse map sampler
uniform sampler2D s_diffuse			: register(S0);
// Beard mask sampler
uniform sampler2D s_beardmask		: register(S1);
// Advanced lighting info sampler
uniform sampler2D lightInfoBuffer	: register(S2);
// Cube map sampler
uniform samplerCUBE cube0			: register(S3);
// Advanced lighting constants
uniform float4 rtParams2;

// Input from vertex shader
struct PS_INPUT
{
	// Texture coordinates
	float2 uv0				: TEXCOORD0;
	// Calculated reflection vector used by the pixel shader to perform a cube map lookup
	float3 reflectVec		: TEXCOORD1;
	// Calculated reflection scale that depends on the relationship between the surface normal and the eye position
	float reflectScale		: TEXCOORD2;
	// Scren space position used for lighting calculations
	float4 screenspacePos	: TEXCOORD3;
	// Normals
	float3x3 viewToTangent	: TEXCOORD4;
};

// Output to the system
struct PS_OUTPUT
{
	float4 color	: COLOR0;
};

PS_OUTPUT main(PS_INPUT IN)
{
	PS_OUTPUT OUT = (PS_OUTPUT)0;
		
	// Maps
	float4 diffuseMap = tex2D( s_diffuse, IN.uv0 );
	float4 beardMap = tex2D( s_beardmask, IN.uv0 );
	float4 cubeMap = texCUBE(cube0, IN.reflectVec);
	diffuseMap *= beardMap;
	diffuseMap.a *= beardMap;
	
	OUT.color = diffuseMap;
	
	// Deferred RT Lighting
	float2 uvScene = IN.screenspacePos.xy / IN.screenspacePos.w;
	uvScene = ( uvScene + 1 ) / 2;
	uvScene.y = 1 - uvScene.y;
	uvScene = ( uvScene * rtParams2.zw ) + rtParams2.xy;
	float3 d_lightcolor;	// Pixel RGB lighting color
	float d_NL_Att;			// Light attenuation
	float d_specular;		// Specular power
	lightinfoUncondition( tex2D(lightInfoBuffer, uvScene), d_lightcolor, d_NL_Att, d_specular);

	// Sub-Surface Approximation
	float4 subSurfaceParams = float4(1,0.6,0.4,0.35);
	float subLamb = smoothstep(-subSurfaceParams.a, 1.0, d_NL_Att) - smoothstep(0.0, 1.0, d_NL_Att);
	subLamb = max(0.0, subLamb);
	OUT.color *= float4( d_lightcolor*0.75 + (subLamb * subSurfaceParams.rgb), 1.0);
	
	// Pixel Specular [Deferred]
	float specularPower = 1;
	float specularStrength = 0.5;
	float specular = pow( d_specular, ceil(specularPower / AL_ConstantSpecularPower)) * specularStrength;
	OUT.color += float4( specularColor.rgb, 0 ) * specular * diffuseMap.a * float4(0.6,0.8,1,1);
	// Cube map fresnel
	OUT.color += cubeMap * diffuseMap.a * (1-IN.reflectScale) * float4(d_lightcolor,1) * 2;
	
	// HDR encode
	OUT.color = hdrEncode( OUT.color );
	
	// Return the output struct to the system
	return OUT;
}

About the author

Hobbyist turned game dev. Worked at DICE for a short while, now working on personal stuff and Dota 2 things.


#1
06/02/2014 (2:14 pm)
Felix,

Nice shader. Best skin shader I have seen thus far. As for the normal issues, I will play around with what you have here and see what I can come up with. In all honesty, what you have right now is FAR beyond the basic stuff we were attempting before so, you would not be remiss in using it as is.

Great job and I look forward to following your progress!

Ron
#2
06/02/2014 (2:27 pm)
Oh wow, that looks excellent. Wish I knew anything about shaders to be able to help out :P.
#3
06/02/2014 (5:26 pm)
Much appreciated Ron! If you are interested in hooking up the cubemap I used I've uploaded it here.

I'm pretty happy with it as of now and think it looks fine even without normal mapping, but I would really really like as well to write a cloth/character shader, and for that normal mapping would be almost essential ;)
#4
06/02/2014 (8:04 pm)
Thanks I will check it out with your cube map and yes, a cloth shader will most certainly require the normal mapping. I will tinker around and see what I can come up with.
#5
06/23/2014 (8:08 pm)
Felix
I can help out some what. I have a working skin shader. It the same one that you'll see in the video
www.youtube.com/watch?v=4Xd8505SMOI
I can talk to Alfio the one that created the shader and see if he has any thing he could suggest. Send me an email jstanleynwo@yahoo.com
#6
03/01/2015 (3:00 pm)
Hey Felix, can you get a hold of me via email scot@alphaeg.com? I don't see a way to reach you in your profile.
#7
03/03/2015 (2:03 am)
Objects that use custom materials also write depth and normals, because things would be rendered wrong.
Render instances are added into multiple render bins (#lightinfo, #prepass, #glowbuffer,...), that output different results. They are rendered more than once and I believe you understand that.
The only exception is the translucent ones. T3D render them forward. But it is not hard fix that.