В общем, проблема такая. Я пытаюсь спарсить список видеокарт с сайта Аvito.com. Я выписал абсолютно все cookies, headers, а также добавил прокси(Есть большой json файлик, но они все бесплатные). Но все же вылетает ошибка 403. Пытаюсь обойти не первый раз, но никак не получатся. Вот код самого парсера.
import requests
from config import *
import json
session = requests.Session()
with open('proxy.json') as file:
proxys_base = json.load(file)['https']
for proxy in proxys_base:
proxys = {
'http://': proxy,
'https://': proxy
}
response = session.get(url=url, headers=headers, cookies=cookies, proxies=proxys, timeout=20)
print(response)
Также в другом файле все данные для запроса.
cookies = {'srv_id': 'QZdbcr7v9YbGAdg9.0bCOd_jLa6Q9tp-clpNEOc7sOriMnmoAMas_u7IBhba-DnJhBrg8cvVAcU0zQj0=.8mffcri3zpjnuDDa4y2spUgpxS1trrhNhgfeN_AdT6I=.web',
'u': '2xzot5hb.qcqjvu.134qn69xjgg00',
'_ym_uid': '168985949341472956',
'_ym_d': '1689859493',
'_ga': 'GA1.1.782121983.1689859493',
'tmr_lvid': 'fec1b449896eddcb9adca146b31ce2b9',
'tmr_lvidTS': '1689859493309',
'uxs_uid': 'cfce9690-2700-11ee-9e21-73703b990417 ',
'adrcid': 'An-DJI0DMTgteYD0_VlcDZQ',
'__zzatw-avito': 'MDA0dBA=Fz2+aQ==',
'auth': '1',
'_buzz_fpc': 'JTdCJTIycGF0aCUyMiUzQSUyMiUyRiUyMiUyQyUyMmRvbWFpbiUyMiUzQSUyMi53d3cuYXZpdG8ucnUlMjIlMkMlMjJleHBpcmVzJTIyJTNBJTIyTW9uJTJDJTIwMTIlMjBBdWclMjAyMDI0JTIwMDklM0E0MiUzQTI3JTIwR01UJTIyJTJDJTIyU2FtZVNpdGUlMjIlM0ElMjJMYXglMjIlMkMlMjJ2YWx1ZSUyMiUzQSUyMiU3QiU1QyUyMnVmcCU1QyUyMiUzQSU1QyUyMmYyOTQ3MWFmYTIzYjEwZWQwMjQwOGVkYjFjNGMwNjgyJTVDJTIyJTJDJTVDJTIyYnJvd3NlclZlcnNpb24lNUMlMjIlM0ElNUMlMjIxMTUuMCU1QyUyMiU3RCUyMiU3RA==',
'cfidsw-avito': '5whPBEn4U3tVObwOkjznkWbNSkSwNxsdgiND2lrlHDy6n9i+L1GUzHOVN+ou7rquhfjLhwPvv7l3qFtr5lBZyZqby58t5VVrRIAfh7buRCjHbaNn76udwUoBImHgydKTNvJpIQhElTyccRZBlXxDNw1PG2m8jRkVoSl+5A==',
'gsscw-avito': '7ES9ZUDZ1x7UbMn4oqXiFRkyyonErVOKRXr7wTm6MLS37Aqpji9Wgwag7VQwH/RG6aj4k8IpUTCFhzY8lPskmTJNdhDMMIvA/AHWYCJRLgOZOIBqiPjRhNkya5ebNv1yRjvaxSFgTfEDxZxIkEf0rw1YnYudaG2lqP96vPnj5kEFfT0NXhDAbER4VUWztDQ+OxpMsvnOPAkzHEZQbHePdpGB+H15gnH6/GRVO/K6ZhHoMNcUK3AKCETbKNOl7g==',
'sessid': '7ES9ZUDZ1x7UbMn4oqXiFRkyyonErVOKRXr7wTm6MLS37Aqpji9Wgwag7VQwH/RG6aj4k8IpUTCFhzY8lPskmTJNdhDMMIvA/AHWYCJRLgOZOIBqiPjRhNkya5ebNv1yRjvaxSFgTfEDxZxIkEf0rw1YnYudaG2lqP96vPnj5kEFfT0NXhDAbER4VUWztDQ+OxpMsvnOPAkzHEZQbHePdpGB+H15gnH6/GRVO/K6ZhHoMNcUK3AKCETbKNOl7g==',
'fgsscw-avito': '53b5755218e8aefb7d411fcbfd194dcf.1691912313',
'cto_bundle': 'tds7NyhOB6C9NIh4H6ofLtd5kDEz5mH0wEH0PYJ6ddZNYpSbkCPUMpsdgwjbK9WrpoiyoSjMS1n3XXg/5aYasM5QgxQJcLaskFIDOYjhX4zFThAlMJbnbgDmsRGNXkYF3mj/Jw9t0wo9sBqRbRrFjFzjVmUFgc/btqI4lQ==',
'_gcl_au': '1.1.1408985415.1698660133',
'buyer_laas_location': '641780',
'_ga_ZJDLBTV49B': 'GS1.1.1702198311.10.0.1702198317.0.0.0',
'_ga_WW6Q1STJ8M': 'GS1.1.1702198311.10.0.1702198317.0.0.0',
'buyer_location_id': '641780',
'yandex_monthly_cookie': 'true',
'abp': '1',
'gMltIuegZN2COuSe': 'EOFGWsm50bhh17prLqaIgdir1V0kgrvN',
'_ym_isad': '1',
'f': '5.1979cf4c013e91892d6059f4e9572c01630247e51b9c7ed6630247e51b9c7ed6630247e51b9c7ed6630247e51b9c7ed6357212485bdbc727357212485bdbc727357212485bdbc72738b4a54cef5443c1d8b16176e03d287314e2ef927eb99aa946b8ae4e81acb9fa1a2a574992f83a9246b8ae4e81acb9fa46b8ae4e81acb9fae992ad2cc54b8aa89a176b7be3f5ad05fa4d7ea84258c63d74c4a16376f87ccd915ac1de0d034112ad09145d3e31a56946b8ae4e81acb9fae2415097439d4047fb0fb526bb39450a46b8ae4e81acb9fa34d62295fceb188dd99271d186dc1cd03de19da9ed218fe2d50b96489ab264edd50b96489ab264edd50b96489ab264ed46b8ae4e81acb9fa38e6a683f47425a8352c31daf983fa077a7b6c33f74d335c03d3f0af72d633b5e9d9fabae5c3e5a402c730c0109b9fbb801bd2c23fa24b0fd59758f902e8f08229aa4cecca288d6b38a6574a5e68337ebed919f123839a5546b8ae4e81acb9fa46b8ae4e81acb9fa02c68186b443a7ac27d63bec7744a8b515872f0e565d0e922da10fb74cac1eab2da10fb74cac1eab25037f810d2d41a8134ecdeb26beb8b53778cee096b7b985bf37df0d1894b088',
'ft': '"DZFyejY1M5hFOK+NLk+wycUKp+M+1adGGylQY6zezO5Y5865OvH4a2tTmp+EMNhg6oq+hvCenj5uNw8tZMxfhfkP6JYOYGgAlpOkTlj+dKioMVcUDlUJKK9/3mFzLpg4TIeJbkBVLfnuOk4dMfHvYQxgj6udEzPmSYt3dk+S5XFhlypgtPBJShRi++AkkALc"',
'v': '1703243430',
'luri': 'novosibirsk',
'_ym_visorc': '0',
'sx': 'b',
'dfp_group': '0',
'SEARCH_HISTORY_IDS': '4%2C%2C0%2C3 ',
'sx': 'H4sIAAAAAAAC%2F5zQO5KjMBAA0LsoJmh9WlL7NpK6ZWxYbAzDb4q7b21A1Uy6F3jB%2B1YQTCpSsAZBLoigRYfiTBSAxAnV7Vst6qYyzbpu%2B8eCvOZjy3OeD96wzZ2rY%2F9QjRJ10wGstdphPBsFVaoEiEBULRQn5FzK3hYfbBQdL7k7uJ8%2F%2FJ5abe4aLePXch%2FM2r2OBKX8lEF7fTbKxExRgjBlk6VUk1xlYLRFO2MDXPIREJIf5HB7TEf%2F2Otz4qi3bmP%2F5fmXDPqf7L33hYOv5Am98yQhiyUOCKUEpkt%2BLrDGcUga3m1dQ38fZRoHPzxn%2BtT38kM2RGTPRuUcjeXoIMYsWNBZnY1JlDXlVF255H159a92QuRH%2B9wen27Z17S%2BYR%2Bnjv50v%2BTo8WxUNdYmcdqw8%2BJMYM5UyEXrUiXw%2F%2FUMCOf5NwAA%2F%2F8GZWDBHQIAAA%3D%3D',
'dfp_group': '25',
'isLegalPerson': '0',
'_ga_M29JC28873': 'GS1.1.1703243541.34.1.1703245263.58.0.0 ',
'tmr_detect': '0%7C1703245269925 ',
'buyer_from_page': 'catalog'
}
headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36 Edg/120.0.0.0',
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',
'Accept-Encoding': 'gzip, deflate, br',
'Accept-Language': 'ru,en;q=0.9,en-GB;q=0.8,en-US;q=0.7',
'Cache-Control': 'max-age=0',
'If-None-Match': 'W/"3a9422-0FyioKtYVRm1zyVYA/VAAXpWxCo"',
'Referer': 'https://www.avito.ru/moskva/tovary_dlya_kompyutera/komplektuyuschie/videokarty-ASgBAgICAkTGB~pm7gmmZw?',
'Sec-Ch-Ua': '"Not_A Brand";v="8", "Chromium";v="120", "Microsoft Edge";v="120"',
'Sec-Ch-Ua-Mobile': '?0',
'Sec-Ch-Ua-Platform': '"Windows"',
'Sec-Fetch-Dest': 'document',
'Sec-Fetch-Mode': 'navigate',
'Sec-Fetch-Site': 'same-origin',
'Sec-Fetch-User': '?1',
'Upgrade-Insecure-Requests': '1'
}
url = 'https://www.avito.ru/novosibirsk/tovary_dlya_kompyutera/komplektuyuschie/videokarty-ASgBAgICAkTGB~pm7gmmZw?cd=1&q=gtx+1660+super'
Файл с прокси(часть)
"http" : [ "1.0.205.87:8080", "1.1.189.58:8080",...
"https" : [ "104.192.224.2","1.0.171.213:8080",...
"socks4" : [ "1.0.136.16:4153", "1.0.136.99:4145"...
"socks5" : [ "1.12.55.136:2080", "1.180.0.162:7302"...
Никак не могу понять в чем проблема
