Am I doing Picking right?

Question

So I'm working on my Picking function. I have read couple of tutorials and read some posts and I got to this point. But it still doesn't work.
What is wrong with my code?

BOOL Directx::Picking(HWND hWnd, AnimatedMesh *entity)
{
    POINT pt;
    D3DVIEWPORT9 vp;
    D3DXMATRIX matWorld, matView, matProj;

    GetCursorPos(&pt);
    ScreenToClient(hWnd, &pt);
    d3ddev->GetTransform(D3DTS_PROJECTION, &matProj);
    d3ddev->GetViewport(&vp);
    d3ddev->GetTransform(D3DTS_WORLD, &matWorld);
    d3ddev->GetTransform(D3DTS_VIEW, &matView);

    // Use inverse of matrix
    D3DXVECTOR3 rayPos(pt.x, pt.y,0); // near-plane position
    D3DXVECTOR3 rayDir(pt.x, pt.x,1); // far-plane position
    D3DXVec3Unproject(&rayPos,&rayPos,&vp,&matProj,&matView,&matWorld);
    D3DXVec3Unproject(&rayDir,&rayDir,&vp,&matProj,&matView,&matWorld);
    rayDir -= rayPos; // make a direction from the 2 positions
    D3DXVec3Normalize(&rayDir,&rayDir);
    // Transform ray origin and direction by inv matrix

    BOOL hasHit;
    float distanceToCollision;

    if(FAILED(D3DXIntersect(entity->pDrawMesh, &rayPos, &rayDir, &hasHit, NULL, NULL, NULL, &distanceToCollision, NULL, NULL)))
    {
        PostQuitMessage(0);
    };

    if(hasHit!=0)
        PostQuitMessage(0);
    else
    {
        s=rayPos;
    }

    return hasHit;
}

Answer 1

Well try this:

BOOL Directx::Picking(HWND hWnd, AnimatedMesh *entity)
{
    POINT pt;
    D3DVIEWPORT9 vp;
    D3DXMATRIX matProj, matView;

    GetCursorPos(&pt);
    ScreenToClient(hWnd, &pt);

    float w = (float)backBufferWidth;
    float h = (float)backBufferHeight;

    d3ddev->GetTransform(D3DTS_VIEW, &matView);
    d3ddev->GetTransform(D3DTS_PROJECTION, &matProj);
    d3ddev->GetViewport(&vp);

    //Transform cursor position to view space
    float x = (2.0f*pt.x/w - 1.0f) / matProj(0,0);
    float y = (-2.0f*pt.y/h + 1.0f) / matProj(1,1);


    D3DXVECTOR3 rayOrigin(0.0f, 0.0f,0.0f); // near-plane position
    D3DXVECTOR3 rayDir(x, y, 1.0f); // far-plane position

    D3DXMATRIX matInvView;

    D3DXMatrixInverse(&matInvView, 0, &matView);

    D3DXVECTOR3 rayOriginW, rayDirW;

    // Transform picking ray to world space.
    D3DXVec3TransformCoord(&rayOriginW, &rayOrigin, &invView);
    D3DXVec3TransformNormal(&rayDirW, &rayDdir, &invView);
    D3DXVec3Normalize(&rayDirW, &rayDirW);

    BOOL hasHit;
    float distanceToCollision;

    if(FAILED(D3DXIntersect(entity->pDrawMesh, &rayOriginW, &rayDirW, &hasHit, NULL, NULL, NULL, &distanceToCollision, NULL, NULL)))
    {
        PostQuitMessage(0);
    };

    s=rayPos;

    return hasHit;
}