Как правильно сделать парсер по страницам dns-shop?

Question

[KolenBG]

def get_dns():
    cookies = {
        'city_path': 'vladivostok',
        'lang': 'ru',
        'current_path': '329c783047da34376e05c68c2340b0dc4c5b4f2a3983a31523e8465bd4d006c5a%3A2%3A%7Bi%3A0%3Bs%3A12%3A%22current_path%22%3Bi%3A1%3Bs%3A144%3A%22%7B%22city%22%3A%2230b7c1ea-03fb-11dc-95ee-00151716f9f5%22%2C%22cityName%22%3A%22%5Cu0412%5Cu043b%5Cu0430%5Cu0434%5Cu0438%5Cu0432%5Cu043e%5Cu0441%5Cu0442%5Cu043e%5Cu043a%22%2C%22method%22%3A%22geoip%22%7D%22%3B%7D',
        '_csrf': '93fd1af79e9ee51c7924f2d56101eaef0d09a7ea30173dfb97e539d3618dbc82a%3A2%3A%7Bi%3A0%3Bs%3A5%3A%22_csrf%22%3Bi%3A1%3Bs%3A32%3A%22Fix6axusbv7QQJZmnaKZGWqDRUNpspTP%22%3B%7D',
        'phonesIdent': '902f4f085ba830bd441029ead13da4d38d946ec1c6d94086db533f6aee4af8f2a%3A2%3A%7Bi%3A0%3Bs%3A11%3A%22phonesIdent%22%3Bi%3A1%3Bs%3A36%3A%2249799e4d-9e9e-4273-b851-193a367bfba4%22%3B%7D',
        'ipp_uid': '1655899145213/XvmE2jCXjDdXoaeP/da+9DQS/lCOZaEhCVfHwSA==',
        '_gcl_au': '1.1.1195741377.1655899147',
        'cartUserCookieIdent_v3': '031585aebe4145ae0b0054b1698f01ca851470b2b2bac0271198612e5c0443fda%3A2%3A%7Bi%3A0%3Bs%3A22%3A%22cartUserCookieIdent_v3%22%3Bi%3A1%3Bs%3A36%3A%22fff84919-44d0-362c-a535-11a34ad2e7ed%22%3B%7D',
        'rcuid': '62b3040ad72cc40001080bbb',
        '_gaexp': 'GAX1.2.F5r0LoWFQuaGkKl-pfLADg.19228.0',
        'tmr_lvid': '65e80dc0dece73ebb3308ecace5e5efe',
        'tmr_lvidTS': '1651741268773',
        '_ym_uid': '1651741269804463153',
        '_ym_d': '1655899148',
        'rrlevt': '1655899338940',
        'rrpvid': '298822378370450',
        'flixgvid': 'flixf67013c5000000.49409826',
        '__ttl__widget__ui': '1656627570645-fdb5fdb10a61',
        'PHPSESSID': 'c3a454ca3c048027803b97dbf08f2b89',
        'rerf': 'AAAAAGLbd8UuKM++BphGAg==',
        '_ab_': 'eb09dad8cccfc9c4ca64bd107efa7a73b3f6538b9282396ad22808fbc3e7385aa%3A2%3A%7Bi%3A0%3Bs%3A4%3A%22_ab_%22%3Bi%3A1%3Ba%3A2%3A%7Bs%3A12%3A%22price-filter%22%3Bs%3A14%3A%22CATALOG_NORMAL%22%3Bs%3A15%3A%22category-filter%22%3Bs%3A6%3A%22simple%22%3B%7D%7D',
        '_gid': 'GA1.2.2088837759.1658909605',
        '_ym_isad': '1',
        'cf_avails': 'now',
        '_ym_visorc': 'b',
        'tmr_detect': '1%7C1659091595517',
        '_ga': 'GA1.2.1970619684.1655899146',
        'tmr_reqNum': '805',
        'dnsauth_csrf': 'e0fad08a34258c3f6ff4450f6b12addb00a57870a4af0248bc0039a5e43bac97a%3A2%3A%7Bi%3A0%3Bs%3A12%3A%22dnsauth_csrf%22%3Bi%3A1%3Bs%3A36%3A%222f825dd2-5750-45d2-866b-cc373dc498e6%22%3B%7D',
        'ipp_key': 'v1659093341919/v33947245ba5adc7a72e273/r9f+tcl/Dk1JcGddEaGPYg==',
        '_ga_FLS4JETDHW': 'GS1.1.1659088097.18.1.1659093345.0',
        'rr-testCookie': 'testvalue',
    }

    headers = {
        'Accept': '*/*',
        'Accept-Language': 'ru-RU,ru;q=0.9,en-US;q=0.8,en;q=0.7',
        'Cache-Control': 'max-age=0',
        'Connection': 'keep-alive',
        # Requests sorts cookies= alphabetically
        # 'Cookie': 'city_path=vladivostok; lang=ru; current_path=329c783047da34376e05c68c2340b0dc4c5b4f2a3983a31523e8465bd4d006c5a%3A2%3A%7Bi%3A0%3Bs%3A12%3A%22current_path%22%3Bi%3A1%3Bs%3A144%3A%22%7B%22city%22%3A%2230b7c1ea-03fb-11dc-95ee-00151716f9f5%22%2C%22cityName%22%3A%22%5Cu0412%5Cu043b%5Cu0430%5Cu0434%5Cu0438%5Cu0432%5Cu043e%5Cu0441%5Cu0442%5Cu043e%5Cu043a%22%2C%22method%22%3A%22geoip%22%7D%22%3B%7D; _csrf=93fd1af79e9ee51c7924f2d56101eaef0d09a7ea30173dfb97e539d3618dbc82a%3A2%3A%7Bi%3A0%3Bs%3A5%3A%22_csrf%22%3Bi%3A1%3Bs%3A32%3A%22Fix6axusbv7QQJZmnaKZGWqDRUNpspTP%22%3B%7D; phonesIdent=902f4f085ba830bd441029ead13da4d38d946ec1c6d94086db533f6aee4af8f2a%3A2%3A%7Bi%3A0%3Bs%3A11%3A%22phonesIdent%22%3Bi%3A1%3Bs%3A36%3A%2249799e4d-9e9e-4273-b851-193a367bfba4%22%3B%7D; ipp_uid=1655899145213/XvmE2jCXjDdXoaeP/da+9DQS/lCOZaEhCVfHwSA==; _gcl_au=1.1.1195741377.1655899147; cartUserCookieIdent_v3=031585aebe4145ae0b0054b1698f01ca851470b2b2bac0271198612e5c0443fda%3A2%3A%7Bi%3A0%3Bs%3A22%3A%22cartUserCookieIdent_v3%22%3Bi%3A1%3Bs%3A36%3A%22fff84919-44d0-362c-a535-11a34ad2e7ed%22%3B%7D; rcuid=62b3040ad72cc40001080bbb; _gaexp=GAX1.2.F5r0LoWFQuaGkKl-pfLADg.19228.0; tmr_lvid=65e80dc0dece73ebb3308ecace5e5efe; tmr_lvidTS=1651741268773; _ym_uid=1651741269804463153; _ym_d=1655899148; rrlevt=1655899338940; rrpvid=298822378370450; flixgvid=flixf67013c5000000.49409826; __ttl__widget__ui=1656627570645-fdb5fdb10a61; PHPSESSID=c3a454ca3c048027803b97dbf08f2b89; rerf=AAAAAGLbd8UuKM++BphGAg==; _ab_=eb09dad8cccfc9c4ca64bd107efa7a73b3f6538b9282396ad22808fbc3e7385aa%3A2%3A%7Bi%3A0%3Bs%3A4%3A%22_ab_%22%3Bi%3A1%3Ba%3A2%3A%7Bs%3A12%3A%22price-filter%22%3Bs%3A14%3A%22CATALOG_NORMAL%22%3Bs%3A15%3A%22category-filter%22%3Bs%3A6%3A%22simple%22%3B%7D%7D; _gid=GA1.2.2088837759.1658909605; _ym_isad=1; cf_avails=now; _ym_visorc=b; tmr_detect=1%7C1659091595517; _ga=GA1.2.1970619684.1655899146; tmr_reqNum=805; dnsauth_csrf=e0fad08a34258c3f6ff4450f6b12addb00a57870a4af0248bc0039a5e43bac97a%3A2%3A%7Bi%3A0%3Bs%3A12%3A%22dnsauth_csrf%22%3Bi%3A1%3Bs%3A36%3A%222f825dd2-5750-45d2-866b-cc373dc498e6%22%3B%7D; ipp_key=v1659093341919/v33947245ba5adc7a72e273/r9f+tcl/Dk1JcGddEaGPYg==; _ga_FLS4JETDHW=GS1.1.1659088097.18.1.1659093345.0; rr-testCookie=testvalue',
        'Origin': 'https://www.dns-shop.ru',
        'Referer': 'https://www.dns-shop.ru/catalog/4e2a7cdb390b7fd7/xolodilniki/no-referrer',
        'Sec-Fetch-Dest': 'empty',
        'Sec-Fetch-Mode': 'cors',
        'Sec-Fetch-Site': 'same-origin',
        'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.0.0 Safari/537.36',
        'X-CSRF-Token': '2SHIkhKpHdfali 24sdfsgg dpCzeJ8-Cr169Y4laUDb5wLrgIRaMifSLCkc9FM5L-j7ROcqCaVRJSgjH_C1Ec9yUXecWE8mA==',
        'X-Requested-With': 'XMLHttpRequest',
        'content-type': 'application/x-www-form-urlencoded',
        'sec-ch-ua': '".Not/A)Brand";v="99", "Google Chrome";v="103", "Chromium";v="103"',
        'sec-ch-ua-mobile': '?0',
        'sec-ch-ua-platform': '"Linux"',
    }

    params = {
        'cityId': '1',
        'langId': 'ru',
        'v': '2',
    }

    data = 'data={"type":"product-buy","containers":[{"id":"as-bfUelZ","data":{"id":"a7eaba54-a869-11e5-9dc3-001e6728a5a4"}},{"id":"as-7ZjF_t","data":{"id":"82ee53de-c58a-11e5-82fc-001e6728a5a4"}},{"id":"as-xc-LtH","data":{"id":"534b5e89-4a06-11e9-ad30-00155d03330f"}},{"id":"as-Efcwry","data":{"id":"bc1a04d4-bd60-11e8-859f-00155d03330f"}},{"id":"as-icY-2V","data":{"id":"2464f40e-dd84-11e8-b10b-00155d03330f"}},{"id":"as-NBgrk_","data":{"id":"7b9e9361-a751-11e8-871d-00155d03330f"}},{"id":"as-M5nC3W","data":{"id":"e2e98938-365f-11e5-a8e5-00155d03361b"}},{"id":"as-yCUwNc","data":{"id":"e1e07eb8-e146-11e6-ab51-00155d03330d"}},{"id":"as-jHFTGs","data":{"id":"48405e0b-e145-11e6-ab51-00155d03330d"}},{"id":"as--ILnKM","data":{"id":"4c4f04c4-105c-11ec-8ee1-00155d8ed20b"}},{"id":"as-7IgWfe","data":{"id":"6dc185b4-2d6f-11eb-a211-00155d03332b"}},{"id":"as-RJDZiW","data":{"id":"bc1a04ce-bd60-11e8-859f-00155d03330f"}},{"id":"as-LXC5GY","data":{"id":"5dcddc77-c41a-11e1-a4dd-001517c526f0"}},{"id":"as-_d8XR3","data":{"id":"8854ae9f-da68-11e3-844d-00155d031202"}},{"id":"as-gLbdg9","data":{"id":"54621700-5585-11e3-b805-005056a615ef"}},{"id":"as-Qnc0gK","data":{"id":"0db77a2e-0486-11e3-a712-00155d030b1f"}},{"id":"as-e7ukmg","data":{"id":"030aa230-aff4-11e4-917d-00155d03361b"}},{"id":"as-rYcc6u","data":{"id":"4ebfa788-88d1-11e9-a20b-00155df1b805"}}]}'

    response = requests.post('https://www.dns-shop.ru/ajax-state/product-buy/', params=params, cookies=cookies,
                             headers=headers, data=data).json()

    items = response.get('data').get('states')

    for item in items:
        name = item.get('data').get('name')
        price = item.get('data').get('price').get('current')

        print(name, price)

Других вариантов не нашел.
Поэтому собрал на POST запросе все ок, все как надо, но при переходе на вторую и последующие страницы переменная data меняется, как это все что я тут выше написал сделать правильно (вроде все просторы интернета прошерстил, но решения не нашел).

P.S. если отправить GET запрос то парсер выдает только скрипты сайта без информации, 'X-Requested-With': 'XMLHttpRequest' в headers вешал менял парсер вариантов нет.

Результат:

Холодильник с морозильником Бирюса 153 белый 21999
Холодильник с морозильником Pozis RS-416 белый 22499
Холодильник с морозильником Pozis Свияга-404-1 белый 22499
Холодильник с морозильником DEXP RF-TD240MA/W серый 22999
Холодильник с морозильником Indesit ITD 125 W белый 22999