Спасибо, вопрос решен - нужно было использовать локальные координаты объекта.
Неправильно:
camdirection = new Vector3(GameObject.FindGameObjectWithTag("MainCamera").GetComponent<Camera>().transform.position.x, GameObject.FindGameObjectWithTag("MainCamera").GetComponent<Camera>().transform.position.y, GameObject.FindGameObjectWithTag("MainCamera").GetComponent<Camera>().transform.position.z);
GameObject.FindGameObjectWithTag("MainCamera").GetComponent<Camera>().transform.position = new Vector3(-camdirection.x, camdirection.y, camdirection.z);
Как надо:
camdirection = new Vector3(GameObject.FindGameObjectWithTag("MainCamera").GetComponent<Camera>().transform.localPosition.x, GameObject.FindGameObjectWithTag("MainCamera").GetComponent<Camera>().transform.localPosition.y, GameObject.FindGameObjectWithTag("MainCamera").GetComponent<Camera>().transform.localPosition.z);
GameObject.FindGameObjectWithTag("MainCamera").GetComponent<Camera>().transform.localPosition = new Vector3(-camdirection.x, camdirection.y, camdirection.z);
