cartoon outlining (cartoon rendering part1)
by F.W. Hardijzer · in Torque Game Engine · 05/31/2003 (3:22 pm) · 113 replies
Ok :)
I've been working on this for about a whole day, and still there are some bugs in it :(
But that doesn't hold me from posting it for others to use :)
so here is the code:
FOR SHAPES
in ts/tsMesh.cc
in TSMesh::render after:
FOR TERRAIN:
in terrain/terrRender.cc
in TerrainRender::renderXFCache after
FOR INTERIORS:
I'm still working on this one, and it's quite difficult (since there are lots of render codes in the interiors,
I've finally gotten some lines over it, but it is pretty glitchy, and will only look good on some interiors,
the $pref::renderOutline for interiors is $Pref::Interior::renderOutline,
and $pref::outlineWidth is $Pref::Interior::outlineWidth...
the reason i changed these vals for interiors is because i can imagine someone wanting terrain and shape, btu not this buggy interior code enabled :)
anyways, enough bullshit, code:
in interior/interiorRender.cc
in void Interior::render
after:
some prefs of this:
$Pref::renderOutline (true/false) render outlines or not
$Pref::outlineWidth (0-10) linewidth of outlines
You are free to use this code for whatever project you'd like, as long as you post every change you make on this forum so others can use it too :)
BUGS:
- enviroment mapped models have 'reflective' lines
Please if you like this stuff, contribute and help us with this!
also if you know anything about cell-shading, try to do that too...
in other words:
FRANK BIGNONE: please post your cell shading stuff from DOP here :(
and another word, noone is allowed to submit this as a resource except for me when i think the code is good enough to be posted as a resource :)
I've been working on this for about a whole day, and still there are some bugs in it :(
But that doesn't hold me from posting it for others to use :)
so here is the code:
FOR SHAPES
in ts/tsMesh.cc
in TSMesh::render after:
S32 drawType = getDrawType(draw.matIndex>>30);
glDrawElements(drawType,draw.numElements,GL_UNSIGNED_SHORT,&indices[draw.start]);
}add/*cell shading*/
bool doOutline=Con::getBoolVariable("$Pref::renderOutline",true);
if (doOutline) {
bool oldlighting=glIsEnabled(GL_LIGHTING);
glDisable(GL_LIGHTING);
F32 oldLineWidth;
int oldcullface;
glGetIntegerv(GL_CULL_FACE,&oldcullface);
glGetFloatv(GL_LINE_WIDTH,&oldLineWidth);
F32 lineWidth=Con::getFloatVariable("$Pref::OutlineWidth",2);
glLineWidth(lineWidth);
glCullFace(GL_FRONT);
glPolygonMode (GL_BACK, GL_LINE);
glDepthFunc(GL_LEQUAL);
bool old2dtex=glIsEnabled(GL_TEXTURE_2D);
glDisable(GL_TEXTURE_2D);
F32 oldcolor[4];
glGetFloatv(GL_CURRENT_COLOR,oldcolor);
glColor4f(0.0f,0.0f,0.0f,1.0f);
for (S32 i=0; i<primitives.size(); i++)
{
TSDrawPrimitive & draw = primitives[i];
AssertFatal(draw.matIndex & TSDrawPrimitive::Indexed,
"TSMesh::render: rendering of non-indexed meshes no longer supported");
S32 drawType = getDrawType(draw.matIndex>>30);
glDrawElements(drawType,draw.numElements,GL_UNSIGNED_SHORT,&indices[draw.start]);
}
glColor4fv(oldcolor);
if (old2dtex) glEnable(GL_TEXTURE_2D);
glDepthFunc(GL_LEQUAL);
glLineWidth(oldLineWidth);
glColor3f(1.0f,1.0f,1.0f);
glCullFace(GL_BACK);
glPolygonMode (GL_BACK, GL_FILL);
if (oldlighting) glEnable(GL_LIGHTING);
}
/* end cell shading*/FOR TERRAIN:
in terrain/terrRender.cc
in TerrainRender::renderXFCache after
U32 vertexCount = mXFIndexBuffer[count + 1];
glDrawElements(mode, vertexCount, GL_UNSIGNED_SHORT, mXFIndexBuffer + count + 2);add://Cell shading
/*cell shading*/
bool doOutline=Con::getBoolVariable("$Pref::renderOutline",true);
if (doOutline) {
/*extra cell shading*/
F32 oldLineWidth;
int oldcullface;
glGetIntegerv(GL_CULL_FACE,&oldcullface);
glGetFloatv(GL_LINE_WIDTH,&oldLineWidth);
F32 lineWidth=Con::getFloatVariable("$Pref::OutlineWidth",2);
glLineWidth(lineWidth);
//glBindTexture(GL_TEXTURE_2D,NULL);
glColor3f(0.0f,0.0f,0.0f);
glCullFace(GL_FRONT);
glPolygonMode (GL_BACK, GL_LINE);
//Draw here
glColor3f(0.0f,0.0f,0.0f);
glDisable(GL_TEXTURE_2D);
glDrawElements(mode, vertexCount, GL_UNSIGNED_SHORT, mXFIndexBuffer + count + 2);
glEnable(GL_TEXTURE_2D);
glLineWidth(oldLineWidth);
glCullFace(GL_BACK);
glPolygonMode (GL_BACK, GL_FILL);
/*end extra cs*/
}
/* end cell shading*/FOR INTERIORS:
I'm still working on this one, and it's quite difficult (since there are lots of render codes in the interiors,
I've finally gotten some lines over it, but it is pretty glitchy, and will only look good on some interiors,
the $pref::renderOutline for interiors is $Pref::Interior::renderOutline,
and $pref::outlineWidth is $Pref::Interior::outlineWidth...
the reason i changed these vals for interiors is because i can imagine someone wanting terrain and shape, btu not this buggy interior code enabled :)
anyways, enough bullshit, code:
in interior/interiorRender.cc
in void Interior::render
after:
if (smRenderMode != 0) {
PROFILE_START(IRO_DebugRender);
debugRender(pMaterials, instanceHandle);
PROFILE_END();
return;
}add://cartoon outline
bool doOutline=Con::getBoolVariable("$Pref::Interior::renderOutline",false);
if (doOutline) {
F32 oldLineWidth=1;
glGetFloatv(GL_LINE_WIDTH,&oldLineWidth);
F32 lineWidth=Con::getFloatVariable("$Pref::Interior::OutlineWidth",2);
glLineWidth(lineWidth);
// Base textures
glBlendFunc(GL_ONE, GL_ZERO);
glDisable(GL_TEXTURE_2D);
glColor3f(0, 0, 0);
for (U32 i = 0; i < sgActivePolyListSize; i++) {
const Surface& rSurface = mSurfaces[sgActivePolyList[i]];
glBegin(GL_LINE_LOOP);
glVertex3fv(mPoints[mWindings[rSurface.windingStart]].point);
S32 skip = rSurface.windingStart + 1;
while (skip < (rSurface.windingStart + rSurface.windingCount)) {
glVertex3fv(mPoints[mWindings[skip]].point);
skip += 2;
}
skip -= 1;
while (skip > rSurface.windingStart) {
if (skip < (rSurface.windingStart + rSurface.windingCount))
glVertex3fv(mPoints[mWindings[skip]].point);
skip -= 2;
}
glEnd();
}
glEnable(GL_TEXTURE_2D);
//return;
}
//End cartoon outlinesome prefs of this:
$Pref::renderOutline (true/false) render outlines or not
$Pref::outlineWidth (0-10) linewidth of outlines
You are free to use this code for whatever project you'd like, as long as you post every change you make on this forum so others can use it too :)
BUGS:
- enviroment mapped models have 'reflective' lines
Please if you like this stuff, contribute and help us with this!
also if you know anything about cell-shading, try to do that too...
in other words:
FRANK BIGNONE: please post your cell shading stuff from DOP here :(
and another word, noone is allowed to submit this as a resource except for me when i think the code is good enough to be posted as a resource :)
About the author
#62
$pref::outlineWidth=(1-10);
Toby, did you get it going, or figure out what the problem is? I have an ati vid chip in my laptop, I can throw it on there and see how it runs? I'd rather not if you already got it working though.
I got new screenies on the website, with the cell shaded effect with better shading... My canyons looked solid, and you couldn't tell where you were.
03/11/2005 (5:07 am)
There is already a variable in the code... It was in teh very first post on this thread.$pref::outlineWidth=(1-10);
Toby, did you get it going, or figure out what the problem is? I have an ati vid chip in my laptop, I can throw it on there and see how it runs? I'd rather not if you already got it working though.
I got new screenies on the website, with the cell shaded effect with better shading... My canyons looked solid, and you couldn't tell where you were.
#63
Find this method in scenegraph\sceneLighting.cc
void SceneLighting::TerrainProxy::lightVector(LightInfo * light)
Replace this:
Point3F lightDir = light->mDirection;
With This:
Point3F lightDir;//= light->mDirection;
lightDir.x = 0.001f;
lightDir.y = 0.001f;
lightDir.z = -80.0f;
There's also an if else blocked (search for "shadowed?" to find it)
It should start like this:
if(height >= intHeight)
{
...
}
else{
...
}
Replace it with this:
// Cell Shading
bool toonColors = Con::getBoolVariable("$Pref::toonColors",true);
bool shadowCheck = true;
if (!toonColors)
{
shadowCheck = height >= intHeight;
}
if(shadowCheck)//height >= intHeight) // non shadowed
{
U32 idx = (xmask + (ymask << blockShift)) << 2;
Point3F normal = pNormals[bi] * normTable[idx++];
normal += pNormals[binext] * normTable[idx++];
normal += pNextNormals[binext] * normTable[idx++];
normal += pNextNormals[bi] * normTable[idx];
normal.normalize();
nextHeightArray[y] = height;
F32 colorScale = -mDot(normal, lightDir);
if (colorScale < 0.0)
{
colorScale = 0.0;
}
if (toonColors)
{
col.red = 5.0 * colorScale;
col.green = 5.0 * colorScale;
col.blue = 5.0 * colorScale;
}
else
{
col += ambient + lightColor * colorScale * lightScale[lsi];
}
// End Cell Shading
}
else // Shadowed
{
nextHeightArray[y] = intHeight;
col += ambient;
}
03/11/2005 (5:27 am)
This is my terrain cell shading code...Find this method in scenegraph\sceneLighting.cc
void SceneLighting::TerrainProxy::lightVector(LightInfo * light)
Replace this:
Point3F lightDir = light->mDirection;
With This:
Point3F lightDir;//= light->mDirection;
lightDir.x = 0.001f;
lightDir.y = 0.001f;
lightDir.z = -80.0f;
There's also an if else blocked (search for "shadowed?" to find it)
It should start like this:
if(height >= intHeight)
{
...
}
else{
...
}
Replace it with this:
// Cell Shading
bool toonColors = Con::getBoolVariable("$Pref::toonColors",true);
bool shadowCheck = true;
if (!toonColors)
{
shadowCheck = height >= intHeight;
}
if(shadowCheck)//height >= intHeight) // non shadowed
{
U32 idx = (xmask + (ymask << blockShift)) << 2;
Point3F normal = pNormals[bi] * normTable[idx++];
normal += pNormals[binext] * normTable[idx++];
normal += pNextNormals[binext] * normTable[idx++];
normal += pNextNormals[bi] * normTable[idx];
normal.normalize();
nextHeightArray[y] = height;
F32 colorScale = -mDot(normal, lightDir);
if (colorScale < 0.0)
{
colorScale = 0.0;
}
if (toonColors)
{
col.red = 5.0 * colorScale;
col.green = 5.0 * colorScale;
col.blue = 5.0 * colorScale;
}
else
{
col += ambient + lightColor * colorScale * lightScale[lsi];
}
// End Cell Shading
}
else // Shadowed
{
nextHeightArray[y] = intHeight;
col += ambient;
}
#64
Also the 'gradient' effect you get on the flat coloured texture is governed by the elevation of the sun, just so you know, if you wanted to get rid of that or tweak it.
I'll get on the changes for the Cel stuff.
Toby.
03/12/2005 (1:42 am)
Chris, Your cel shading works, but if you turn it off then the terrain disapears - Have you tried that?Also the 'gradient' effect you get on the flat coloured texture is governed by the elevation of the sun, just so you know, if you wanted to get rid of that or tweak it.
I'll get on the changes for the Cel stuff.
Toby.
#65
After thinking about it... I don't see why the height of the sun affects it. I mean I'm not even using the sun in my calculations. I'm using a directional light, which I have specified the direction for.
03/13/2005 (8:46 am)
Where did you find that out? I've been trying to get rid of it for ever, but couldn't figure out how. I have tried almost everything I could think of to get rid of it. I will look into it, or let me know if you get rid of it. That must be in another peice of code somewhere.After thinking about it... I don't see why the height of the sun affects it. I mean I'm not even using the sun in my calculations. I'm using a directional light, which I have specified the direction for.
#66
Toby.
03/13/2005 (8:48 am)
Well essentially I doubt you'll get rid of it, without messing with the terrain lighting - However, you can put the sun to be directly over the map which means a pretty flat shade.Toby.
#68
03/13/2005 (9:05 am)
I'm not using the sun... I have specified my own direction for the light source. And the direction cannot be straight down, or the function exits... And I am messing with the terrain lighting...
#69
}
else
lightScale[lsi] = 0.f;
}
// Cell Shading
bool toonColors = Con::getBoolVariable("$Pref::toonColors",true);
bool shadowCheck = true;
if (!toonColors)
{
shadowCheck = height >= intHeight;
}
if(shadowCheck)//height >= intHeight) // non shadowed
{
U32 idx = (xmask + (ymask << blockShift)) << 2;
Point3F normal = pNormals[bi] * normTable[idx++];
normal += pNormals[binext] * normTable[idx++];
normal += pNextNormals[binext] * normTable[idx++];
normal += pNextNormals[bi] * normTable[idx];
normal.normalize();
nextHeightArray[y] = height;
F32 colorScale = -mDot(normal, lightDir);
if (colorScale < 0.0)
{
colorScale = 0.0;
}
if (toonColors)
{
col.red = 5.0 * colorScale;
col.green = 5.0 * colorScale;
col.blue = 5.0 * colorScale;
}
else
{
col += ambient + lightColor * colorScale * lightScale[lsi];
}
// End Cell Shading
}
else // Shadowed
{
nextHeightArray[y] = intHeight;
col += ambient;
}
}
F32 * tmp = heightArray;
heightArray = nextHeightArray;
nextHeightArray = tmp;
}
// set the proper color
for(i = 0; i < TerrainBlock::LightmapSize * TerrainBlock::LightmapSize; i++)
{
mLightmap[i] *= inverseRatioSquared;
mLightmap[i].clamp();
}
delete [] normTable;
delete [] heightArray;
delete [] nextHeightArray;
Where in code is the whole gradient effect controlled? I managed to flat shade it completely.. I have screenies of it on the webpage. It still has a gradient, it's just really unnoticeable unless you get close. I want to remove it completely, and put a very slight texture on everything... I think that will look best.
03/13/2005 (6:20 pm)
Toby, My terrain doesn't disappear when I turn off toonColors. It just goes to the lighting it would normally do on the terrain. . . Here is what I am using...}
else
lightScale[lsi] = 0.f;
}
// Cell Shading
bool toonColors = Con::getBoolVariable("$Pref::toonColors",true);
bool shadowCheck = true;
if (!toonColors)
{
shadowCheck = height >= intHeight;
}
if(shadowCheck)//height >= intHeight) // non shadowed
{
U32 idx = (xmask + (ymask << blockShift)) << 2;
Point3F normal = pNormals[bi] * normTable[idx++];
normal += pNormals[binext] * normTable[idx++];
normal += pNextNormals[binext] * normTable[idx++];
normal += pNextNormals[bi] * normTable[idx];
normal.normalize();
nextHeightArray[y] = height;
F32 colorScale = -mDot(normal, lightDir);
if (colorScale < 0.0)
{
colorScale = 0.0;
}
if (toonColors)
{
col.red = 5.0 * colorScale;
col.green = 5.0 * colorScale;
col.blue = 5.0 * colorScale;
}
else
{
col += ambient + lightColor * colorScale * lightScale[lsi];
}
// End Cell Shading
}
else // Shadowed
{
nextHeightArray[y] = intHeight;
col += ambient;
}
}
F32 * tmp = heightArray;
heightArray = nextHeightArray;
nextHeightArray = tmp;
}
// set the proper color
for(i = 0; i < TerrainBlock::LightmapSize * TerrainBlock::LightmapSize; i++)
{
mLightmap[i] *= inverseRatioSquared;
mLightmap[i].clamp();
}
delete [] normTable;
delete [] heightArray;
delete [] nextHeightArray;
Where in code is the whole gradient effect controlled? I managed to flat shade it completely.. I have screenies of it on the webpage. It still has a gradient, it's just really unnoticeable unless you get close. I want to remove it completely, and put a very slight texture on everything... I think that will look best.
#70
Have you made any recent changes? How's the performance on your card?
Toby.
03/31/2005 (4:15 pm)
Chris, I'm getting our head programmer to check your code out, however, both on ATI 8500 and Nvidia Geforce 3, the terrain disapears if you turn off the toon outlines.Have you made any recent changes? How's the performance on your card?
Toby.
#71
04/08/2005 (5:17 am)
I know how to fix that little bug. However I quit working on toon shaders. The bug is in the if (toonColors) condition... there is no else{ // draw}, which there should be. I will post the working code tonight if you'd like. It requires finding it. I hope I still have it kicking around. How has development been going on toon shaders for you?
#72
04/08/2005 (5:17 am)
There really isn't any performance overhead.
#73
We're cracking on with things and I'll update this thread once we've made progress.
Thanks again!
Toby.
04/08/2005 (5:40 am)
Chris, If you don't mind psoting the fix, that would be much appreciated!We're cracking on with things and I'll update this thread once we've made progress.
Thanks again!
Toby.
#74
04/08/2005 (6:59 am)
No, prob... Just give me a few hours
#75
This doesn't happen, or only to a very minimal extent on my ATI 8500 All-In-Wonder. When we had our devathon, this problem occured on most Nvidia cards, but the FX.
Just to share. I'll be updating here with a couple of nice Cel-Shaded pictures soon.
Toby.
04/10/2005 (12:50 pm)
There are definately some strong driver-linked problems, I'm at home with a Geforce 3 Ti 500, and I get a lot of problems with Cel-Outlines appearing in the 'middle' of my mesh. This is using the latest Nvidia drivers. This doesn't happen, or only to a very minimal extent on my ATI 8500 All-In-Wonder. When we had our devathon, this problem occured on most Nvidia cards, but the FX.
Just to share. I'll be updating here with a couple of nice Cel-Shaded pictures soon.
Toby.
#76
Outline fix:
void TerrainRender::renderXFCache()
{
U32 count = 0;
bool doOutline=Con::getBoolVariable("$Pref::renderOutline",true);
F32 oldLineWidth;
glGetFloatv(GL_LINE_WIDTH,&oldLineWidth);
F32 lineWidth=Con::getFloatVariable("$Pref::OutlineWidth",2);
while (count < mXFIndexCount)
{
U32 mode = mXFIndexBuffer[count];
U32 vertexCount = mXFIndexBuffer[count + 1];
// Cartoon Outline
if(doOutline)
{
glHint (GL_LINE_SMOOTH_HINT, GL_NICEST); // Use The Good Calculations
glEnable (GL_LINE_SMOOTH);
glPolygonMode (GL_BACK, GL_LINE);
glLineWidth(lineWidth);
glDisable(GL_CULL_FACE);
glColor3f(0.0f,0.0f,0.0f);
glDisable(GL_TEXTURE_2D);
glDrawElements(mode, vertexCount, GL_UNSIGNED_SHORT, mXFIndexBuffer + count + 2);
glEnable(GL_TEXTURE_2D);
glEnable(GL_CULL_FACE);
glLineWidth(oldLineWidth);
glPolygonMode (GL_BACK, GL_FILL);
glDrawElements(mode, vertexCount, GL_UNSIGNED_SHORT, mXFIndexBuffer + count + 2);
}
else
{
glDrawElements(mode, vertexCount, GL_UNSIGNED_SHORT, mXFIndexBuffer + count + 2);
}
// End Cartoon Outline
count += vertexCount + 2;
}
}
That will fix the problem... I am using a NVidia card and ATI cards... There are no problems with either... if you know how it works, all it does is draw a line on edges that have one poly dfacing you and one facing away. Therefore, if your mesh is messy and has a lot of polys in it with large angles between them then you will see a lot of lines. If you are using TSE (which makes your life easier) then just write a shader (or find one online) and use that instead.
It is buggy for ATI cards... Your programmer will have to find a way to do it for ATI then use OpenGL device queries to see if it is ATI or NVidia...
Let me know if you have anymore problems...
Hope this helped
04/10/2005 (5:26 pm)
Im using NVidia and have no problems... Outline fix:
void TerrainRender::renderXFCache()
{
U32 count = 0;
bool doOutline=Con::getBoolVariable("$Pref::renderOutline",true);
F32 oldLineWidth;
glGetFloatv(GL_LINE_WIDTH,&oldLineWidth);
F32 lineWidth=Con::getFloatVariable("$Pref::OutlineWidth",2);
while (count < mXFIndexCount)
{
U32 mode = mXFIndexBuffer[count];
U32 vertexCount = mXFIndexBuffer[count + 1];
// Cartoon Outline
if(doOutline)
{
glHint (GL_LINE_SMOOTH_HINT, GL_NICEST); // Use The Good Calculations
glEnable (GL_LINE_SMOOTH);
glPolygonMode (GL_BACK, GL_LINE);
glLineWidth(lineWidth);
glDisable(GL_CULL_FACE);
glColor3f(0.0f,0.0f,0.0f);
glDisable(GL_TEXTURE_2D);
glDrawElements(mode, vertexCount, GL_UNSIGNED_SHORT, mXFIndexBuffer + count + 2);
glEnable(GL_TEXTURE_2D);
glEnable(GL_CULL_FACE);
glLineWidth(oldLineWidth);
glPolygonMode (GL_BACK, GL_FILL);
glDrawElements(mode, vertexCount, GL_UNSIGNED_SHORT, mXFIndexBuffer + count + 2);
}
else
{
glDrawElements(mode, vertexCount, GL_UNSIGNED_SHORT, mXFIndexBuffer + count + 2);
}
// End Cartoon Outline
count += vertexCount + 2;
}
}
That will fix the problem... I am using a NVidia card and ATI cards... There are no problems with either... if you know how it works, all it does is draw a line on edges that have one poly dfacing you and one facing away. Therefore, if your mesh is messy and has a lot of polys in it with large angles between them then you will see a lot of lines. If you are using TSE (which makes your life easier) then just write a shader (or find one online) and use that instead.
It is buggy for ATI cards... Your programmer will have to find a way to do it for ATI then use OpenGL device queries to see if it is ATI or NVidia...
Let me know if you have anymore problems...
Hope this helped
#77
04/11/2005 (3:16 am)
Thanks for your help so far Chris, I'll be looking into this this week and will post back any changes made :)
#78
04/11/2005 (3:31 am)
Great. I wish you luck. If you need any help with the engine modifications, I can probably spare a small amount of time. I have both an ATI and a NVidia card for testing, so I can help in that respect as well.
#79
04/14/2005 (10:59 am)
How'd the engine modifications go?
#80
Iven been looking through OpenGl Tuts and they do it the same way, but dont have that issue.
04/17/2005 (11:40 am)
Just wondering if anyone solved the line issue with shapes?Iven been looking through OpenGl Tuts and they do it the same way, but dont have that issue.
Torque Owner Toby W. Allen
Toby.