Вам нужно написать свой JsonDeserializer и указать его в аннотации @JsonDeserialize над тем сеттером поля с которым у вас проблемы
import com.fasterxml.jackson.core.JsonParser;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.DeserializationContext;
import com.fasterxml.jackson.databind.JsonDeserializer;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
import java.io.IOException;
public class TestJackson {
	Integer a;
	public Integer getA() {
		return a;
	}
	@JsonDeserialize(using = StringIntegerDeserializer.class)
	public void setA(Integer a) {
		this.a = a;
	}
	@Override
	public String toString() {
		return "TestJackson{a=" + a + "}";
	}
	public static void main(String[] args) throws IOException {
		ObjectMapper mapper = new ObjectMapper();
		String json = "{\"a\":\"5.666\"}";
		System.out.println(mapper.readValue(json, TestJackson.class));
		json = "{\"a\":5.666}";
		System.out.println(mapper.readValue(json, TestJackson.class));
		json = "{\"a\":5}";
		System.out.println(mapper.readValue(json, TestJackson.class));
	}
	public static class StringIntegerDeserializer extends JsonDeserializer<Integer> {
		@Override
		public Integer deserialize(JsonParser jsonParser, DeserializationContext deserializationContext) throws IOException, JsonProcessingException {
			String valueAsString = jsonParser.getValueAsString();
			int integer = (int) Double.parseDouble(valueAsString);
			return integer;
		}
	}
}