This is a common problem with using matrix multiplication to compose all three rotations i would recommend to use either your own camera class that uses angle yaw and pitch that define forward vector, and up vector definition from which you can calculate right vector then you just apply these three to view matrix.

Now whys ur acting like that? imagine you roll camera, you will see that only up and right vector get changed.

But still that should be doable with only applying transformation to unit vec right?

I guess youll have to transform vector that you have initialized at the beggining(first func)

So you delete

s->current_cam->right = orientation[0]; s->current_cam->up = orientation[1]; s->current_cam->vec = orientation[2];

And apply rotation matrix to initial values defined in first function.

But that wont work… anyway

why is your cam up vector 0,0,0

Ill give you a 6dof code so you can read it

This is not my code you get it as review not for use

void pitch(float cs, float sn)
{
   float temp;

   temp = ay*cs - az*sn;
   az   = ay*sn + az*cs;
   ay = temp;

   temp = by*cs - bz*sn;
   bz   = by*sn + bz*cs;
   by = temp;

   temp = cy*cs - cz*sn;
   cz   = cy*sn + cz*cs;
   cy = temp;
}
void yaw(float cs, float sn)
{
   float temp;

   temp = az*cs - ax*sn;
   ax   = az*sn + ax*cs;
   az = temp;

   temp = bz*cs - bx*sn;
   bx   = bz*sn + bx*cs;
   bz = temp;

   temp = cz*cs - cx*sn;
   cx   = cz*sn + cx*cs;
   cz = temp;
}
void roll(float cs, float sn)
{
   float temp;

   temp = ax*cs - ay*sn;
   ay   = ax*sn + ay*cs;
   ax = temp;

   temp = bx*cs - by*sn;
   by   = bx*sn + by*cs;
   bx = temp;

   temp = cx*cs - cy*sn;
   cy   = cx*sn + cy*cs;
   cx = temp;
}
Now try to dot it
3dpoint<float>  mmul(t3dpoint<float> adr_ptr)
{
t3dpoint<float> p;
   p.x = ax*adr_ptr.x + ay* adr_ptr.y  + az* adr_ptr.z;
   p.y = bx*adr_ptr.x + by* adr_ptr.y  + bz* adr_ptr.z;
   p.z = cx*adr_ptr.x + cy* adr_ptr.y  + cz* adr_ptr.z;
return p;
}

void DoRotation()
{

rf =	  mmul(		vec3(0,0-1)	  	);
ru =	  mmul(		vec3(0,1,0)	  		);
rr =	  mmul(		vec3(0,01)	  	);
}

As you see in this code in order to apply proper ttansformation you change not all parts of rotation vector.

Anyway i lost the track.

Consider

X pointing right

Y up

Z forward

Basically you just pitch and yaw camera you do not roll it

You apply pitch transformation that changes only y and z

You yaw camera that changes x and z

You compose a rotation vector (more alike rot matrix)

Anyway noone knows what type of camera will suffice your needs i would go for simple fpp cam with pitch and yaw angles defined which will compose forward vector you can then compose up vector knowing that (i may be wrong) vec3 up(somex, somey, somez) (from initial up vector values you define in forst func) and you add it to forward vec (vec3 newup = forward + up*100);

vec3 right = normalize(front x newup);

newup = front x right;//may be in wrong order

Here is my cam class (not ideal)

#ifndef SpectatorH
#define SpectatorH

#include "/mnt/ext_sdcard/WNB/globalvars.cpp"
#include "/mnt/ext_sdcard/WNB/DxcMath.h"
#include "/mnt/ext_sdcard/WNB/PhysicalElement.h"

struct TSpecCamera : public PComponent<float, float, float, float, float, float>
{
public:
//t3dpoint<float> pos;
t3dpoint<float> old;
t3dpoint<float> upvec;
t3dpoint<float> lookat;

Matrix44<float> LOOK_MAT;

bool lookatpoint;
t3dpoint<float> lap;

float glop; float heading;//says from which angles we started to see
float speed;
float yawa;
float pitcha;
float rolla;
t3dpoint<float> FORWARD_LOOK_AT;
t3dpoint<float> Xvec; t3dpoint<float> Yvec; t3dpoint<float> Zvec;
float pitchy;
typrNota rot;

bool target;

t3dpoint<float> tmp; //to not make additional variables even when they are deleted after func call



t3dpoint<float> GetPos() { return pos; }

void SetCamera( t3dpoint<float> face_first_vert, t3dpoint<float> face_sec_vert, t3dpoint<float> face_last_vert, t3dpoint<float> normal)
{
Xvec = Normalize(vectorAB(face_first_vert,face_sec_vert));
Zvec = Normalize(vectorAB(face_first_vert,face_last_vert));

//pitch
Yvec = normal;
float cosp = cos(pitcha*imopi);
double x = 1000.0*sin(yawa*imopi)*cosp;
double y = 1000.0*sin(pitcha*imopi);
double z = 1000.0*cos(yawa*imopi)*cosp;


FORWARD_LOOK_AT = ((Xvec * x) - (Yvec * y) + (Zvec * z));// + (div2 * y);

}

void yaw(float angle)
{
yawa = angle;
if (angle    > 360  )    yawa = 0;
if (angle    < -360 )    yawa = 0;

}

void pitch(float angle)
{
pitcha = angle;

if (angle    > heading+85.0  )    pitcha = heading+85.0;
if (angle    < heading-85.0  )    pitcha = heading-85.0;
}

void move_mouse(float x, float y)
{
	if (!lookatpoint) move_mouse_FPP(x, y);
	else
		move_mouse_target();
}
void move_mouse_target()
{
//	double qx = 1000.0*cos(yawa*imopi)*cos(pitcha*imopi);
//	double qy = 1000.0*sin(pitcha*imopi);
//	double qz = 1000.0*sin(yawa*imopi)*cos(pitcha*imopi);

pos = lap - Normalize( FORWARD_LOOK_AT )*80.0;
}

void move_mouse_FPP(float x, float y)
{

	glop 	= glop - (LastMousePos.x - x)/100.0f * MouseSpeed;
	heading = heading - (LastMousePos.y - y)/100.0f * MouseSpeed;

	yaw(glop);
	pitch(heading);

	if (heading > 90.0   ) heading = 90.0;
	if (heading < -90.0  ) heading = -90.0;

	if (glop    > 360  )    glop = 0;
	if (glop    < -360 )    glop = 0;

}

void GoForward(float dt)
{
tmp = Normalize( FORWARD_LOOK_AT );
pos = pos + tmp* speed*dt;
}

void GoBackward(float dt)
{
tmp = Normalize( FORWARD_LOOK_AT );
pos = pos - tmp* speed*dt;
}

void GoStrafeLeft(float dt)
{
double x = 1000.0*cos((yawa+90.0)*imopi);
double z = 1000.0*sin((yawa+90.0)*imopi);


tmp = ((Xvec * x) + (Zvec * z));
tmp = Normalize( tmp );

pos = pos - tmp* speed*dt;
}

void GoStrafeRight(float dt)
{
double x = 1000.0*cos((yawa-90.0)*imopi);
double z = 1000.0*sin((yawa-90.0)*imopi);


tmp = ((Xvec * x) + (Zvec * z));
tmp = Normalize( tmp );

pos = pos - tmp* speed*dt;
}


t3dpoint<float> ReturnRightVector()
{
	float x = 1000.0*cos((yawa-90.0)*imopi);
	float z = 1000.0*sin((yawa-90.0)*imopi);
tmp = ((Xvec * x) + (Zvec * z));

return Normalize( tmp );
}

t3dpoint<float> ReturnFrontVector()
{
double x = 1000.0*cos(yawa*imopi)*cos(pitcha*imopi);
double y = 1000.0*sin(pitcha*imopi);
double z = 1000.0*sin(yawa*imopi)*cos(pitcha*imopi);


FORWARD_LOOK_AT = ((Xvec * x) - (Yvec * y) + (Zvec * z));

return Normalize( FORWARD_LOOK_AT );
}

t3dpoint<float> ReturnUpVector() {  return ReturnRightVector() * ReturnFrontVector(); }


void SetView()
{
double x = 1000.0*cos(yawa*imopi)*cos(pitcha*imopi);
double y = 1000.0*sin(pitcha*imopi);
double z = 1000.0*sin(yawa*imopi)*cos(pitcha*imopi);


FORWARD_LOOK_AT = ((Xvec * x) - (Yvec * y) + (Zvec * z));// + (div2 * y);

lookat = pos + FORWARD_LOOK_AT;

if (!lookatpoint)
glLookAt(LOOK_MAT, pos, lookat, Yvec);
else
glLookAt(LOOK_MAT, pos, lap, Yvec);



ACTUAL_VIEW = LOOK_MAT;
}
  //http://img5.demotywatoryfb.pl//uploads/201410/1413481018_by_Prawe_Udo_inner.gif winter


TSpecCamera()
{
upvec 	= t3dpoint<float>(0.0,1.0,0.0);
pos 	= t3dpoint<float>(0.0,0.0,0.0);
Xvec 	= t3dpoint<float>(1.0,0.0,0.0);
Yvec 	= t3dpoint<float>(0.0,1.0,0.0);
Zvec 	= t3dpoint<float>(0.0,0.0,-1.0);
pitcha  = 0.0;
yawa    = 0.0;
speed = 0.0;
glop = 0.0;
heading = 0.0;
lookatpoint = false;
target = false;
}

~TSpecCam