Александр

struct Plane {
    int px1 = 0;
    int px2 = 0;
    int py1 = 0;
    int py2 = 0;
};

Plane getPlane(const QPolygon &shape) {
    Plane plane;
    plane.px1 = shape.at(0).x();
    plane.px2 = plane.px1;
    plane.py1 = shape.at(0).y();
    plane.py2 = plane.py1;

    foreach (const QPoint &point, shape) {
        if(point.x() < plane.px1) plane.px1 = point.x();
        if(point.x() > plane.px2) plane.px2 = point.x();
        if(point.y() < plane.py1) plane.py1 = point.y();
        if(point.y() > plane.py2) plane.py2 = point.y();
    }

    return plane;
}

void test()
{
    QPolygon shapeA;
    shapeA.append(QPoint(100,100));
    shapeA.append(QPoint(120,90));
    shapeA.append(QPoint(150,110));
    shapeA.append(QPoint(150,130));
    shapeA.append(QPoint(120,140));
    shapeA.append(QPoint(90,130));

    QPolygon shapeB;
    shapeB.append(QPoint(100,120));
    shapeB.append(QPoint(110,110));
    shapeB.append(QPoint(140,120));
    shapeB.append(QPoint(120,130));
    shapeB.append(QPoint(100,190));

    Plane planeA = getPlane(shapeA);
    Plane planeB = getPlane(shapeB);

    if(planeA.px1 < planeB.px1
            && planeA.px2 > planeB.px2
            && planeA.py1 < planeB.py1
            && planeA.py2 > planeB.py2) {
        qDebug() << "Фигура B в фигуре A!";
    }
    else if(planeA.px1 > planeB.px1
            && planeA.px2 < planeB.px2
            && planeA.py1 > planeB.py1
            && planeA.py2 < planeB.py2) {
        qDebug() << "Фигура A в фигуре В!";
    }
    else {
        /* nothing */
    };
}