RenderTexture with mixed 2d and 3d nodes

[martinking71]

Hi,

I’m trying to create a torch like effect and also a magnifying effect using shaders and have had some success using RenderTexture.

All works fine until I add in 3d nodes to the mix.

I keep my own scene graph and call visit as needed on Nodes - mainly ax::Sprites and ax::Labels. My scene graph is composed of many 2d nodes with potentially many local 3d scene nodes.

I keep all my local MeshRenderer Nodes in a local Scene with their own Camera and before calling render on this local 3d Scene I call render() on the Director’s renderer to flush it and render any previous 2d nodes - saves any depth settings for 2d. This then renders this local 3d node and allows more 2d nodes to be rendered on top of it and more local 3d scenes as need be, simply layering nodes over each other.

Is there a way to flush the RenderTexture or how should I go about extending RenderTexture to support it. The end() command swaps the original target back in so rendering doesn’t go to the texture anymore. Calling render on the Director's renderer before rendering these local 3d nodes just produces a black screen as RenderTexture::end() needs to be called first. And if end() is called before rendering a local 3d scene I only get up to that point rendered into the texture so the shader effect only works on part of the scene.

I feel like I’m missing an obvious call to clear a depth buffer or something similar instead of the call to end().

And again just to reiterate, this process works fine if I don’t render to a texture or if I do but just to 2d Nodes - the manual render, the local 3d scenes with their cameras and collections of MeshRenderers - but obviously I can’t create the effects I want using shaders, which is the whole point of this exercise.

Thanks for any pointers.

[rh101]

It's a little hard to test without an example to run, but have you considered implementing your own version of end(), where it sets a different callback than onEnd(), one that doesn't change the render target?

For example, this is how they're currently implemented:

void RenderTexture::end()
{
    Renderer* renderer = _director->getRenderer();

    auto endCommand = renderer->nextCallbackCommand();
    endCommand->init(_globalZOrder);
    endCommand->func = AX_CALLBACK_0(RenderTexture::onEnd, this);

    renderer->addCommand(endCommand);
    renderer->popGroup();

    _director->popMatrix(MATRIX_STACK_TYPE::MATRIX_STACK_PROJECTION);
    _director->popMatrix(MATRIX_STACK_TYPE::MATRIX_STACK_MODELVIEW);
}

void RenderTexture::onEnd()
{
    _director->loadMatrix(MATRIX_STACK_TYPE::MATRIX_STACK_PROJECTION, _oldProjMatrix);
    _director->loadMatrix(MATRIX_STACK_TYPE::MATRIX_STACK_MODELVIEW, _oldTransMatrix);

    Renderer* renderer = _director->getRenderer();
    renderer->setViewPort(_oldViewport.x, _oldViewport.y, _oldViewport.width, _oldViewport.height);

    renderer->setRenderTarget(_oldRenderTarget);
}

How about implementing and using these for the "layers" that you described:

RenderTextureEx* RenderTextureEx::create(int w, int h, ax::backend::PixelFormat format, ax::backend::PixelFormat depthStencilFormat, bool sharedRenderTarget)
{
    auto* ret = new RenderTextureEx();

    if (ret->initWithWidthAndHeight(w, h, format, depthStencilFormat, sharedRenderTarget))
    {
        ret->autorelease();
        return ret;
    }
    AX_SAFE_DELETE(ret);
    return nullptr;
}

RenderTextureEx* RenderTextureEx::create(int w, int h, ax::backend::PixelFormat format, bool sharedRenderTarget)
{
    auto* ret = new RenderTextureEx();

    if (ret->initWithWidthAndHeight(w, h, format, sharedRenderTarget))
    {
        ret->autorelease();
        return ret;
    }
    AX_SAFE_DELETE(ret);
    return nullptr;
}

RenderTextureEx* RenderTextureEx::create(int w, int h, bool sharedRenderTarget)
{
    auto* ret = new RenderTextureEx();

    if (ret->initWithWidthAndHeight(w, h, backend::PixelFormat::RGBA8, PixelFormat::NONE, sharedRenderTarget))
    {
        ret->autorelease();
        return ret;
    }
    AX_SAFE_DELETE(ret);
    return nullptr;
}

void RenderTextureEx::endPartial()
{
    Renderer* renderer = _director->getRenderer();

    auto endCommand = renderer->nextCallbackCommand();
    endCommand->init(_globalZOrder);
    endCommand->func = AX_CALLBACK_0(RenderTextureEx::onEndPartial, this);

    renderer->addCommand(endCommand);
    renderer->popGroup();

    _director->popMatrix(MATRIX_STACK_TYPE::MATRIX_STACK_PROJECTION);
    _director->popMatrix(MATRIX_STACK_TYPE::MATRIX_STACK_MODELVIEW);
}

void RenderTextureEx::onEndPartial()
{
    _director->loadMatrix(MATRIX_STACK_TYPE::MATRIX_STACK_PROJECTION, _oldProjMatrix);
    _director->loadMatrix(MATRIX_STACK_TYPE::MATRIX_STACK_MODELVIEW, _oldTransMatrix);

    Renderer* renderer = _director->getRenderer();
    renderer->setViewPort(_oldViewport.x, _oldViewport.y, _oldViewport.width, _oldViewport.height);

    //renderer->setRenderTarget(_oldRenderTarget);
}

void RenderTextureEx::onBeginPartial()
{
    _oldProjMatrix = _director->getMatrix(MATRIX_STACK_TYPE::MATRIX_STACK_PROJECTION);
    _director->loadMatrix(MATRIX_STACK_TYPE::MATRIX_STACK_PROJECTION, _projectionMatrix);

    _oldTransMatrix = _director->getMatrix(MATRIX_STACK_TYPE::MATRIX_STACK_MODELVIEW);
    _director->loadMatrix(MATRIX_STACK_TYPE::MATRIX_STACK_MODELVIEW, _transformMatrix);

    if (!_keepMatrix)
    {
        _director->setProjection(_director->getProjection());
        const Vec2& texSize = _texture2D->getContentSizeInPixels();

        // Calculate the adjustment ratios based on the old and new projections
        Vec2 size         = _director->getWinSizeInPixels();
        float widthRatio  = size.width / texSize.width;
        float heightRatio = size.height / texSize.height;

        Mat4 orthoMatrix;
        Mat4::createOrthographicOffCenter((float)-1.0 / widthRatio, (float)1.0 / widthRatio, (float)-1.0 / heightRatio,
                                          (float)1.0 / heightRatio, -1, 1, &orthoMatrix);
        _director->multiplyMatrix(MATRIX_STACK_TYPE::MATRIX_STACK_PROJECTION, orthoMatrix);
    }

    Rect viewport;
    viewport.size.width           = _fullviewPort.size.width;
    viewport.size.height          = _fullviewPort.size.height;
    float viewPortRectWidthRatio  = float(viewport.size.width) / _fullRect.size.width;
    float viewPortRectHeightRatio = float(viewport.size.height) / _fullRect.size.height;
    viewport.origin.x             = (_fullRect.origin.x - _rtTextureRect.origin.x) * viewPortRectWidthRatio;
    viewport.origin.y             = (_fullRect.origin.y - _rtTextureRect.origin.y) * viewPortRectHeightRatio;

    Renderer* renderer = _director->getRenderer();

    _oldViewport = renderer->getViewport();
    renderer->setViewPort(viewport.origin.x, viewport.origin.y, viewport.size.width, viewport.size.height);

    //_oldRenderTarget = renderer->getRenderTarget();
    //renderer->setRenderTarget(_renderTarget);
}

void RenderTextureEx::beginPartial()
{
    _director->pushMatrix(MATRIX_STACK_TYPE::MATRIX_STACK_PROJECTION);
    _projectionMatrix = _director->getMatrix(MATRIX_STACK_TYPE::MATRIX_STACK_PROJECTION);

    _director->pushMatrix(MATRIX_STACK_TYPE::MATRIX_STACK_MODELVIEW);
    _transformMatrix = _director->getMatrix(MATRIX_STACK_TYPE::MATRIX_STACK_MODELVIEW);

    if (!_keepMatrix)
    {
        _director->setProjection(_director->getProjection());

        const Vec2& texSize = _texture2D->getContentSizeInPixels();

        // Calculate the adjustment ratios based on the old and new projections
        Vec2 size = _director->getWinSizeInPixels();

        float widthRatio  = size.width / texSize.width;
        float heightRatio = size.height / texSize.height;

        Mat4 orthoMatrix;
        Mat4::createOrthographicOffCenter((float)-1.0 / widthRatio, (float)1.0 / widthRatio, (float)-1.0 / heightRatio,
                                          (float)1.0 / heightRatio, -1, 1, &orthoMatrix);
        _director->multiplyMatrix(MATRIX_STACK_TYPE::MATRIX_STACK_PROJECTION, orthoMatrix);
    }

    Renderer* renderer = _director->getRenderer();
    auto* groupCommand = renderer->getNextGroupCommand();
    groupCommand->init(_globalZOrder);
    renderer->addCommand(groupCommand);
    renderer->pushGroup(groupCommand->getRenderQueueID());

    auto beginCommand = renderer->nextCallbackCommand();
    beginCommand->init(_globalZOrder);
    beginCommand->func = AX_CALLBACK_0(RenderTextureEx::onBeginPartial, this);
    renderer->addCommand(beginCommand);
#if AX_ENABLE_CACHE_TEXTURE_DATA
    _cachedTextureDirty = true;
#endif
}

So it would be used this way:

rt->beginWithClear(0, 0, 0, 0);

// first layer
// ..visit nodes here
rt->endPartial();

Director::getInstance()->getRenderer()->clear(ClearFlag::DEPTH, Color4F::BLACK, depth, 0, rt->getGlobalZOrder()); // Everything ignored except depth value
rt->beginPartial();
// ..visit nodes here
rt->endPartial();

// Final layer here
Director::getInstance()->getRenderer()->clear(ClearFlag::DEPTH, Color4F::BLACK, depth, 0, rt->getGlobalZOrder());
rt->beginPartial();
// ..visit nodes here

rt->end(); // the final call to the original end()

Just taking a guess here, so I'm not sure if this would work for what you're after, but it a quick test with 2D nodes, it does render them correctly to a single RenderTexure.

[rh101]

@martinking71 Is this effect what you're after?

This is with a sepia shader applied to the RenderTexture sprite:

The updated code in my previous post is what was used for this. It consists of a 2D layer, then a 3D layer, then a 2D layer on top, all rendered onto the same RenderTexture.

[martinking71]

That's it!

Was just about to upload an extension to the cpp-tests that showed the same thing! I should have created the test first to make it easier for you to see.

Thanks for all your hard work and insight

[rh101]

Excellent, I'm glad it worked. Also, yes, a sample project would have helped a lot, and something to keep in mind for the future.

There may be a way to add this functionality directly into the engine, and avoid having to introduce new methods etc... Perhaps a flag passed to RenderTexure::begin() and RenderTexture::end(), which will dictate whether it should call setRenderTarget or not.

[aismann]

Yes, an example on the cpp-tests will be great too

[aismann]

Was just about to upload an extension to the cpp-tests that showed the same thing! I should have created the test first to make it easier for you to see.

@martinking71
Your welcome to add your example to cpp-tests. It can help for future issues. thanks.

[rh101]

@aismann Once the functionality is implemented in RenderTexture, then a test will be added as well. I've tried out several different ways of adding the functionality directly into RenderTexture, and while they all work, they are not very intuitive, so I'll see if I can work out a better way.

For the time being, the easiest way is just to sub-class RenderTexture and add the functionality as shown in my post above.

[martinking71]

Not sure if this helps but I've attached my rough work in progress RenderTextureTest and updated CMakelists.txt file. Should just drop into the cpp-tests and run on test set 42. Click the menu to cycle through the different render states - normal, normal with flush, to texture, to texture with flush.

I can't seem to get the overridden render fnc to call its ancestor correctly when rendering to texture, it crashes - not sure why this is. (see the top of the render fnc where the ancestor is called only when not rendering to texture). Also obvious when the menu no longer renders during the render to texture states

Not sure if this is something to do with rendering a scene when inside a scene render already?

RenderTextureTest.zip
CMakeLists.txt

[rh101]

When you're trying out new features, it's perhaps best to add them to a new project, just to ensure no other code is running that may impact the implementation. So, I suggest you create a new project, add the relevant code to it, and then see if it works.

To significantly cut down on build time, add the following to the new project CMakeLists.txt, just after the project(${APP_NAME}) line:

set(AX_EXT_HINT OFF CACHE BOOL "The default extensions hint" FORCE)
# Enable any of the following if required, then re-run the axmol build command with the -f parameter to ensure cached values are updated
#set(AX_ENABLE_EXT_SPINE ON CACHE BOOL "Build extension spine" FORCE)
#set(AX_ENABLE_EXT_DRAGONBONES ON CACHE BOOL "Build extension DragonBones" FORCE)
#set(AX_ENABLE_EXT_COCOSTUDIO ON CACHE BOOL "Build extension cocostudio" FORCE)
#set(AX_ENABLE_EXT_FAIRYGUI ON CACHE BOOL "Build extension FairyGUI" FORCE)
#set(AX_ENABLE_EXT_ASSETMANAGER ON CACHE BOOL "Build extension asset-manager" FORCE)
#set(AX_ENABLE_EXT_PARTICLE3D ON CACHE BOOL "Build extension Particle3D" FORCE)
#set(AX_ENABLE_EXT_LUA ON CACHE BOOL "Build lua libraries" FORCE)
#set(AX_ENABLE_EXT_GUI ON CACHE BOOL "Build extension GUI" FORCE)
#set(AX_ENABLE_EXT_JSONDEFAULT ON CACHE BOOL "Build extension JSONDefault" FORCE)
#set(AX_ENABLE_EXT_PHYSICS_NODE ON CACHE BOOL "Build extension physics-nodes" FORCE)
set(AX_ENABLE_3D ON CACHE BOOL "Build 3D support" FORCE)

set(AX_ENABLE_PHYSICS OFF CACHE BOOL "Build Physics3D support" FORCE)
set(AX_ENABLE_3D_PHYSICS OFF CACHE BOOL "Build Physics3D support" FORCE)

set(AX_ENABLE_NAVMESH OFF CACHE BOOL "Build NavMesh support" FORCE)
set(AX_ENABLE_AUDIO OFF CACHE BOOL "" FORCE) # Since this is disabled, set the USE_AUDIO_ENGINE defined in AppDelegate.cpp to 0
set(AX_ENABLE_HTTP OFF CACHE BOOL "Build HTTP client based on yasio" FORCE)
set(AX_ENABLE_WEBSOCKET OFF CACHE BOOL "Build Websocket client based on yasio" FORCE)
set(AX_WITH_BULLET OFF CACHE BOOL "" FORCE)

[martinking71]

renderTextureTest.zip

Here is a cleaner stand alone project that shows the different visit() processing and the effects using the extended RenderTexture with the partialBegin / Ends.

Just click on the menu option to see:

• normal visit with no flush - wrong draw order
• normal visit with flush - correct draw order
• render to texture with no flush - 3d not rendered to texture but default target? (see the renderTexture slide away to reveal)
• render to texture with flush - 3d rendered out of position.

There are a few movement effects added to show some of the issues I've found when rendering to texture. Simple new project so should be easy to spot the mistakes I've probably made. All the work is in the MixedMode::visit() where all visits() are manually called.

I would guess that the camera might not be set correctly. It looks like it's just using the default camera - you have to have a camera with ax::CameraFlag::USER1 set so the 3d renders but it seems to ignore the other camera values - FOV, depth and transform. In the MixedMode constructor there is some commented out code that should make the camera move but doesn't. A block below will make the scene move so that shows updates are working, just not on the camera.

[rh101]

Just FYI, the project crashes on the 3rd of 4th click because of an invalid index:

I just changed it to the following in order to get it working:

void MixedModeTest::normal(Object* sender)
{
    static const char* messages[] = {
        "render: default - wrong order",
        "render: default with flush - correct",
        "render: to texture - missing 3d",
        "render: to texture with flush - 3d off centre"
    };

    _renderMode = static_cast<eRenderMode>((_renderMode + 1) % std::size(messages));

    _renderStateLabel->setString(messages[_renderMode]);
}

[rh101]

• render to texture with flush - 3d rendered out of position.

Can you please show a screenshot of what it should look like when it's at the correct position, since there is nothing to compare it with in the demo.

[martinking71]

My bad zipped up out of date file

[martinking71]

The second setting in the demo is the correct one - all the versions should look this was - see the visit() fnc with the manual draw order as all Nodes have the default depth settings so the draw order should be as ordered. The flushing should ensure this - the without flushes are there so show what else happens.

1. Tinted red sprite grossini
2. 3d model over the top centred as above
3. Un-tinted sprite grossinis-sister1 on top of that.

Thanks again for looking into this

[rh101]

Just a suggestion, each test case should be self-contained, and with the minimal amount of code to reproduce the issue, making it much easier to see exactly what is going on without jumping around test code. Attached are modified files with the changes to allow for this.

RenderTextureTestModified.zip

The output of the tests:
Test 1:

Test 2:

Test 3:

Test 4:

From what you described, 2 of the tests above show the required output, being tests 2 and 4. Tests 1 and 3 do not show the required output, but that is expected given the implementations do not support your requirements, so there is no actual bug there, unless there is something I'm misunderstanding.

If you think there is some bug, then please be very specific as to what it is, and show what output you actual expect to see, and what you are actually seeing.

[martinking71]

Here's what I see when running my code before and also with your updates.

I see the same as you for 1 & 2 with 2 being the effect I expected - manual draw order but the renderer->render() must be called to get the 3d drawn in between the sprites - because the sprites and 3d have have default depth settings.
I guess 3 is the same - the sliding render texture was to show that the 3d was rendered to the default render target not the RenderTexture as expected, but again guessing that's why the flush is needed.
In 4 you can see that the 3d is rendered just off top right on my machine.

I'm running on a MacBook Pro M2 if that makes any difference.

[martinking71]

Yes,

Running the camera action should be producing the same effect - or pretty close to it - it's the same action in effect.

[rh101]

Out of curiosity, have you tested the action on a camera when using the default scene tree? What I mean by that is don't do the visit calls yourself, but just add the nodes to the scene, and let the engine do the processing. If so, does the action on the camera work in that case?

[martinking71]

It does, which suggests that I'm missing something else out when I call the visit() manually.

[martinking71]

How did you get the sprite drawn, then the 3d and then another sprite when rendering to texture in your original test? Did you manually visit the nodes?

[rh101]

The test I initially created is using the following code (with resources taken from the cpp-tests project):

Add these to the class in MainScene.h:

ax::RefPtr<RenderTextureEx> _rtEx;
ax::RefPtr<ax::MeshRenderer> _mesh;
ax::RefPtr<ax::RepeatForever> _swim;
ax::RefPtr<ax::Animate3D> _hurt;
State _state;
ax::RefPtr<ax::MoveTo> _moveAction;
uint64_t s_sepia_program_id;
ax::RefPtr<ax::backend::ProgramState> _sepiaState;

In MainScene.cpp:

bool MainScene::init()
{
    if (!Scene::init())
    {
        return false;
    }

    scheduleUpdate();

    auto programManager = ProgramManager::getInstance();
    s_sepia_program_id  = programManager->registerCustomProgram(positionTextureColor_vert, "custom/example_Sepia_fs"sv,
                                                                VertexLayoutType::Sprite);

    _rtEx = RenderTextureEx::create(visibleSize.width, visibleSize.height);
    addChild(_rtEx, -1);
    _rtEx->setPosition(visibleSize / 2);
    _rtEx->setAnchorPoint(Vec2::ANCHOR_MIDDLE);

    std::string fileName = "tortoise.c3b";
    _mesh = MeshRenderer::create(fileName);
    _mesh->setScale(0.2f);
    auto s = Director::getInstance()->getWinSize();
    _mesh->setPosition(Vec2(s.width * 4.f / 5.f, s.height / 2.f - 30));

    auto animation = Animation3D::create(fileName);
    if (animation)
    {
        auto animate = Animate3D::create(animation, 0.f, 1.933f);
        _swim = RepeatForever::create(animate);
        _mesh->runAction(_swim);

        _hurt = Animate3D::create(animation, 1.933f, 2.8f);
        _state = State::SWIMMING;
    }

    _moveAction = MoveTo::create(4.f, Vec2(s.width / 5.f, s.height / 2.f - 30));

    auto seq = Sequence::create(_moveAction, CallFunc::create(AX_CALLBACK_0(MainScene::reachEndCallBack, this)), nullptr);
    seq->setTag(100);
    _mesh->runAction(seq);

    _sepiaState = createSepiaProgramState();
    _sepiaState->updateBatchId();

    return true;
}

void MainScene::onEnter()
{
    Scene::onEnter();
    _mesh->onEnter();
}

void MainScene::onExit()
{
    _mesh->onExit();
    Scene::onExit();
}

ax::backend::ProgramState* MainScene::createSepiaProgramState()
{
    auto programState = new backend::ProgramState(ProgramManager::getInstance()->loadProgram(s_sepia_program_id));
    programState->autorelease();
    return programState;
}

void MainScene::update(float delta)
{
    _rtEx->beginWithClear(1, 0, 0, 1);
    
    auto sprite = Sprite::create("HelloWorld.png"sv);
    sprite->setPosition(_rtEx->getContentSize() / 2);
    sprite->visit();
    _rtEx->endPartial();
    
    // Director::getInstance()->getRenderer()->clear(ClearFlag::DEPTH, Color4F::BLACK, 10, 0,
    //                                               _rtEx->getGlobalZOrder());
    
    _rtEx->beginPartial();
    _mesh->visit(Director::getInstance()->getRenderer(), Mat4::IDENTITY, 0);
    _rtEx->endPartial();
    
    // Director::getInstance()->getRenderer()->clear(ClearFlag::DEPTH, Color4F::BLACK, 10, 0,
    //                                               _rtEx->getGlobalZOrder());
    
    _rtEx->beginPartial();
    auto sprite2 = Sprite::create("HelloWorld.png"sv);
    sprite2->setPosition(_rtEx->getContentSize() / 2 - Vec2(100, 100));
    sprite2->visit();
    
    _rtEx->end();
    _rtEx->getSprite()->setProgramState(_sepiaState); // apply the Sepia shader
}