select distinct user_from as user_id from message where user_to = 123
union distinct
select distinct user_to as user_id from message where user_from = 123
select
distinct if(user_from=123,user_to,user_from) as user_id
from `message `
where user_to = 123 OR user_from = 123