Silhouette selection via postFx for Torque3D

#1 For this to work with Beta 5, please replace your render function with the following - (the zipped resource was not updated to reflect this change)

void RenderSelectionMgr::render( SceneState *state )  
{  
   PROFILE_SCOPE( RenderSelectionMgr_Render );  

   // Don't allow non-diffuse passes.  
   if ( !state->isDiffusePass() )  
      return;  
  
   GFXDEBUGEVENT_SCOPE( RenderSelectionMgr_Render, ColorI::GREEN );  
  
   GFXTransformSaver saver;  
  
   // Tell the superclass we're about to render, preserve contents  
   const bool isRenderingToTarget = _onPreRender( state );  
  
   // Clear all the buffers to black.  
   GFX->clear( GFXClearTarget, ColorI::BLACK, 1.0f, 0);  
  
   // init loop data  
   SceneGraphData sgData;  
   U32 binSize = mElementList.size();  
  
   for( U32 j=0; j<binSize; )  
   {  
      MeshRenderInst *ri = static_cast<MeshRenderInst*>(mElementList[j].inst);  
  
      setupSGData( ri, sgData );  
      sgData.binType = SceneGraphData::SelectionBin;  
  
      BaseMatInstance *mat = ri->matInst;  
  
      U32 matListEnd = j;  
  
      while( mat->setupPass( state, sgData ) )  
      {  
         U32 a;  
         for( a=j; a<binSize; a++ )  
         {  
            MeshRenderInst *passRI = static_cast<MeshRenderInst*>(mElementList[a].inst);  
  
            if (newPassNeeded(mat, passRI))  
               break;  
  
            mat->setTransforms(*passRI->objectToWorld, *passRI->worldToCamera, *passRI->projection);  
            mat->setEyePosition(*passRI->objectToWorld, state->getCameraPosition());  
            mat->setBuffers(passRI->vertBuff, passRI->primBuff);  
  
            if ( passRI->prim )  
               GFX->drawPrimitive( *passRI->prim );  
            else  
               GFX->drawPrimitive( passRI->primBuffIndex );  
         }  
         matListEnd = a;  
      }  
  
      // force increment if none happened, otherwise go to end of batch  
      j = ( j == matListEnd ) ? j+1 : matListEnd;  
   }  
  
   // Finish up.  
   if ( isRenderingToTarget )  
      _onPostRender();  
}

#2
The .cpp file : PART 1

#include "KSelectRenderBin.h"
#include "console/consoleTypes.h"
#include "materials/shaderData.h"
#include "gfx/gfxPrimitiveBuffer.h"
#include "gfx/gfxTransformSaver.h"
#include "math/util/matrixSet.h"
#include "gfx/gfxShader.h"
#include "materials/sceneData.h"
#include "postFx/postEffectManager.h"
#include "gfx/gfxTarget.h"
#include "gfx/gfxTextureHandle.h"
#include "core/stream/fileStream.h"
#include "gui/buttons/guiButtonCtrl.h"

#include "materials/matTextureTarget.h"

#include "sceneGraph/sceneState.h"

#include "windows.h"


IMPLEMENT_CONOBJECT(KSelectRenderBin);


KSelectRenderBin::KSelectRenderBin()
{
	assignName("KSelectBin");

	ShaderData *SD;
	ShaderData *SSH;
	Sim::findObject( "KSelect", SD );
	Sim::findObject( "KDiffuse", SSH );
	mSelect		= SD->getShader();
	mDiffuse	= SSH->getShader();

	mconstants		= mSelect->allocConstBuffer();
	mSELECTIONTEX	= new GFXTexHandle;
	mDEPTH			= new GFXTexHandle;
	mNOISE			= new GFXTexHandle;
	mNOISE->set( "core/scripts/client/postFx/noise.png", &GFXDefaultStaticDiffuseProfile, "Noise for highlight overlay" ); 

	mVIEWPROJ = mSelect->getShaderConstHandle("$viewproj");
	mSIZE = mSelect->getShaderConstHandle("$size"); 

	mTarget = GFX->allocRenderToTextureTarget();

	md.cullDefined		= true;
	md.cullMode			= GFXCullCCW;
	md.samplersDefined	= true;
	md.samplers[0]		= GFXSamplerStateDesc::getWrapLinear();
	md.setBlend( false );
	md.setZReadWrite( true, true );
}

KSelectRenderBin::KSelectRenderBin(RenderInstType riType, F32 renderOrder, F32 processAddOrder)
   : RenderBinManager(riType, renderOrder, processAddOrder)
{
	assignName("KSelectBin");
	ShaderData *SD;
	ShaderData *SSH;
	Sim::findObject( "KSelect", SD );
	Sim::findObject( "KDiffuse", SSH );
	mSelect		= SD->getShader();
	mDiffuse	= SSH->getShader();

	mconstants		= mSelect->allocConstBuffer();
	mSELECTIONTEX	= new GFXTexHandle;
	mDEPTH			= new GFXTexHandle;
	mNOISE			= new GFXTexHandle;
	mNOISE->set( "core/scripts/client/postFx/noise.png", &GFXDefaultStaticDiffuseProfile, "Noise for highlight overlay" ); 

	mVIEWPROJ = mSelect->getShaderConstHandle("$viewProj"); 
	mSIZE = mSelect->getShaderConstHandle("$size");

	mTarget = GFX->allocRenderToTextureTarget();

	md.cullDefined		= true;
	md.cullMode			= GFXCullCCW;
	md.samplersDefined	= true;
	md.samplers[0]		= GFXSamplerStateDesc::getWrapLinear();
	md.setBlend( false );
	md.setZReadWrite( true, true );
}

KSelectRenderBin::~KSelectRenderBin()
{
	SAFE_DELETE( mSELECTIONTEX );
	SAFE_DELETE( mDEPTH );
	SAFE_DELETE( mSELECTIONTEX );
	SAFE_DELETE( mNOISE );
}

void KSelectRenderBin::init()
{

}

void KSelectRenderBin::initPersistFields()
{
	Parent::initPersistFields();
}

#3 The cpp file : PART TWO

void KSelectRenderBin::render(SceneState* state)
{
	//-> Not familiar with a way to receive GFX events yet.  IE: Resize.
	GFXTarget* BB = GFX->getActiveRenderTarget();
	mSELECTIONTEX->set( BB->getSize().x, BB->getSize().y, BB->getFormat(), &GFXDefaultRenderTargetProfile, "Kyle Color Texture" );
	mDEPTH->set( BB->getSize().x, BB->getSize().y, GFXFormatD24S8, &GFXDefaultZTargetProfile, "Kyle Depth Texture" );

	//-> Clear our render target.  Cant do it incrimentally.
	//-> Wish they had a clear function inside of textures.
	//-> Is attaching textures every frame necessary? Might only need
	//-> to do it once.
	GFX->pushActiveRenderTarget();
		mTarget->attachTexture( GFXTextureTarget::Color0, *mSELECTIONTEX );
		mTarget->attachTexture( GFXTextureTarget::DepthStencil, *mDEPTH );
		GFX->setActiveRenderTarget( mTarget );
		GFX->clear( GFXClearZBuffer | GFXClearStencil | GFXClearTarget, ColorI(0,0,0), 1.0f, 0 );
	GFX->popActiveRenderTarget();

	for( U32 j = 0; j < mElementList.size(); ++j)
	{
		MeshRenderInst *ri = static_cast<MeshRenderInst*>(mElementList[j].inst);
		BaseMatInstance *mi = ri->matInst;

		GFXTransformSaver saver;

		//-> Setup render target.
		GFX->pushActiveRenderTarget();
			mTarget->attachTexture( GFXTextureTarget::Color0, *mSELECTIONTEX );
			mTarget->attachTexture( GFXTextureTarget::DepthStencil, *mDEPTH );
			GFX->setActiveRenderTarget( mTarget );
			
			//-> Have the objects material setup the pass (best thing about it is diffuse textures are set).
			SceneGraphData SGD;
			setupSGData( ri, SGD );
			mi->setupPass(state, SGD);
			mi->setBuffers(ri->vertBuff, ri->primBuff);

			//-> Calculate viewproj matrix and screen size ...
			MatrixF VIEWPROJ(*ri->projection);
			VIEWPROJ *= *ri->worldToCamera;
			VIEWPROJ *= *ri->objectToWorld;
			Point2F SIZE( BB->getSize().x, BB->getSize().y );

			//-> Set shader constants
			mconstants->setSafe( mVIEWPROJ, VIEWPROJ );
			mconstants->setSafe( mSIZE, SIZE );
			GFX->setShaderConstBuffer(mconstants);

			//-> State block
			GFX->setStateBlockByDesc( md );

			//-> Set shader and draw
			GFX->setShader( mDiffuse );
			GFX->drawPrimitive( ri->primBuffIndex );

		GFX->popActiveRenderTarget();	
	}

	for( U32 j = 0; j < mElementList.size(); ++j)
	{
		//-> Same as above, but am doing selection pass.
		MeshRenderInst *ri = static_cast<MeshRenderInst*>(mElementList[j].inst);
		BaseMatInstance *mi = ri->matInst;
		SceneGraphData SGD;
		setupSGData( ri, SGD );
		mi->setupPass(state, SGD);
		mi->setBuffers(ri->vertBuff, ri->primBuff);
		MatrixF VIEWPROJ(*ri->projection);
		VIEWPROJ *= *ri->worldToCamera;
		VIEWPROJ *= *ri->objectToWorld;
		mconstants->setSafe( mVIEWPROJ, VIEWPROJ );
		Point2F SIZE( BB->getSize().x, BB->getSize().y );
		mconstants->setSafe( mSIZE, SIZE );
		GFX->setShaderConstBuffer(mconstants);
		GFX->setStateBlockByDesc( md );
		GFX->setTexture( 3, *mSELECTIONTEX );
		GFX->setTexture( 4, *mNOISE );
		GFX->setShader(mSelect);
		GFX->drawPrimitive( ri->primBuffIndex );
	}
}

RenderBinManager::AddInstResult KSelectRenderBin::addElement( RenderInst *inst )
{
	if (inst->translucentSort)
		return RenderBinManager::arSkipped;

	if( inst->type == RenderInstType("Mesh") )
	{
		internalAddElement(inst);
		return RenderBinManager::arAdded;
	}
	else
		return RenderBinManager::arSkipped;
}

#4 NOTE: I removed my selection method, since i had to modify quite a few classes to get it working. The addElement function would filter elements, of course if you choose to implement a method.

Next - shaders:

Youll need to add shader definitions somewhere, similar to this. If you rename them, make sure to change the name in the .cpp:

singleton ShaderData( KSelect )
{   
   DXVertexShaderFile 	= "shaders/common/postFx/Kyle/kSelectV.hlsl";
   DXPixelShaderFile 	= "shaders/common/postFx/Kyle/kSelectP.hlsl";
      
   pixVersion = 3.0;
};


singleton ShaderData( KDiffuse )
{   
   DXVertexShaderFile 	= "shaders/common/postFx/Kyle/kDiffuseV.hlsl";
   DXPixelShaderFile 	= "shaders/common/postFx/Kyle/kDiffuseP.hlsl";
      
   pixVersion = 3.0;
};

The four shader are as follows:

kDiffuseV.hlsl:

struct VIn
{
	float4 pos  : POSITION;
	float4 col	: COLOR;
	float2 uv   : TEXCOORD0;
	float3 tang	: TANGENT; 
};

struct VertToPix
{
	float4 pos  : POSITION;
	float4 col	: COLOR;
	float2 uv0	: TEXCOORD0;
};


VertToPix main( VIn IN, uniform float4x4 viewProj)
{
	VertToPix R;
	float3 T 	= IN.tang;
	R.col 		= IN.col;
	R.pos 		= mul(viewProj, IN.pos);
	R.uv0 		= IN.uv;
	
	return R;
}

#5
kDiffuseP.hlsl

struct VertToPix
{
	float4 newpos 	: VPOS;
	float4 pos  	: POSITION;
	float4 col		: COLOR;
	float2 uv0		: TEXCOORD0;
};


float4 main( VertToPix IN, uniform sampler2D diffuseMap : register(S0), uniform sampler2D BBT : register(S4)) : COLOR
{
	float4 C1 = tex2D( diffuseMap, IN.uv0 );	
	C1.a = 1;
	return C1;
}

kSelectV.hlsl

struct VS_OUT
{
	float4 pos 		: POSITION;
	float2 uv0 		: TEXCOORD0;
	float4 color 	: COLOR;
};

struct VIN
{
  	float4 pos        	: POSITION;
	float4 color	  	: COLOR;
	float2 uv0        	: TEXCOORD0;
	float3 tangent		: TANGENT;
	float3 normal		: NORMAL;
};


VS_OUT main( VIN IN, uniform float4x4 viewProj)
{
	VS_OUT R;	
	
	R.color = IN.color;
	R.uv0 	= IN.uv0;
	R.pos = mul( viewProj, IN.pos );
	
	return R;
}

#6 kSelectP.hlsl

struct PX_IN
{
	float2 position : VPOS;
	float4 color : COLOR;
	float2 uv0 : TEXCOORD0;
};


float4 main( PX_IN IN, uniform sampler2D diffuse : register(S0),
			uniform sampler2D selectionBuffer : register(S3),
			uniform sampler2D noise : register(S4),
			uniform float2 size,
			float2 hPos : VPOS) : COLOR
{
	int OFFSET = 3;

	   float2 offsets[9] = {  
      float2( 0.0,  0.0),  
      float2(-OFFSET, -OFFSET),  
      float2( 0.0, -OFFSET),  
      float2( OFFSET, -OFFSET),  
      float2( OFFSET,  0.0),  
      float2( OFFSET,  OFFSET),  
      float2( 0.0,  OFFSET),  
      float2(-OFFSET,  OFFSET),  
      float2(-OFFSET,  0.0),  
   };
   
   float2 PixelSize = 1.0 / size;////float2(1024,768);  
	
	float avgval = 0;
	
	for(int i = 0; i < 9; i++)  
	{  
		float2 uv = (hPos + offsets[i]) * PixelSize;  
		float4 cpix = float4( tex2D( selectionBuffer, uv ).rrr, 1.0 );
		avgval += clamp(cpix.r*256, 0, 1);
	}
	
	avgval /= 11;
	
	float vis = round(1.0-(abs(frac(avgval)-0.5)*2));
	
	float4 bb	= tex2D(selectionBuffer, hPos * PixelSize);
	float4 e	= float4(vis, 0, 0, vis);
	float4 ovr	= float4(avgval, 0, 0, avgval);
	
	ovr *= 0.5;

	bb = lerp(bb, ovr, ovr.a);
	e = lerp(bb, e, e.a);
	
	float4 COLOR = tex2D( noise, IN.uv0*(size/(size/50)) );
	float NOISE_VAL = dot( COLOR, float4(0.3, 0.59, 0.18, 0));
	e += NOISE_VAL/5;
	
	return e;
}

Final thing to do. Add the noise image in. You will need to change the path in the .cpp file, to wherever you put the image file... here is the image:

http://img338.imageshack.us/img338/6180/noisea.png

The final result should look something like this:


If you have any questions, throw em at me. Thanks again for the thread.

#7 Thanks Kyle!

So you don't necessarily need a noise texture, as it will just be an overlay, right? What's the bonus with using a noise sample instead of just the color? Or were you just looking for that specific effect?

Also it doesn't seem so much different than the original resource - I might be mistaken about that though since I just skimmed the code. What's the part that would increase performance? Perhaps it could be changed for the original resource too.

And thanks again, really cool of you to share the entire stuff.

#8 Yea the noise isn't necessary, I was going to add an animation to it with a few random numbers since the artist I was working with (Greg G) wanted animated static.

I never meant that there was a definite increase in speed, I havn't really tested it vs. any other method. From what I skimmed over though, it seems that the post effect class sets a few extra variables and makes a few calls/allocations that aren't necessary for an effect like this. This method could be much faster if:

1) There was a way to only allocate the textures when screen size changed. Right now, its allocating 2 full screen textures every frame. Perhaps set width/height variables and check them in the render function where ->set(...) is being called right now. If different, reallocate.

2) A way to clear buffers without having to push/pop/set them. Perhaps i just dont know about how torque does this. Lines 12-17 of the .cpp.

3) If the set shader constants were pulled out of the loops.

Forget the others I had thought of...

Anyway, currently most models increase the render time of this by about 0.4-0.6ms per model. With 7 models on the screen, taking up about half the screen space, I was getting render times of about 3-4ms. I'm guessing with other optimizations it would drop a small amount? Like I said though, don't have much to compare it to at the moment :(

#9 Other thing is - with this method there is an outline overlap. I didn't look into it to fix it, and it isn't really a problem for what were doing. But if you have two models that overlap each other in the world, their combined texture is outlined, since the outline is applied after diffuse objects are rendered to the selection texture. This could be addressed a number of ways, but currently the above code does not.

#10 How to change the line thickness?

#11 Line thickness is based on how far from the current pixel you are sampling from ... I put the general offset into a variable.

int OFFSET = 3;

Increase/decrease that if you want to change the thickness of the line.

#12 Kyle, thanks for reply

#13 I'm not sure what's up, but i think your site is down. Think you could email me the files? Admin@Tandune.com.

Thanks.

#14 Thanks for letting me know, Kenny. It's our NS provider who has an unscheduled maintenance. Please use this link instead.

#15 Konrad, any chance for this resource to be ported to 1.1b3?

#16 I've been meaning to do that for a while now. I definitely want to do it, but I might not be able to get to it for another week or two. Sorry about that, things have been busy around here.

#17 Still, good news it's not dead. Looking forward to this moment to come =)

I've spent an hour or two today rewriting it and trying to wire back sticking ends, compiled it with no errors, but didn't manage to make it work. So I guess I'll leave it to those who dig rendering code better.

#18

Quote:
... So I guess I'll leave it to those who dig rendering code better.

I know. We should ask someone. :)

#19 Any news about this?
Should we live and hope? =)