Ошибка 403 при парсинге сайта?

Question

[Alenwar]

Пробую забрать данные с интернет магазина и выскакивает ошибка 403

import requests
from bs4 import BeautifulSoup
from time import sleep
from random import choice


desktop_agents = ['Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.99 Safari/537.36',
                 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.99 Safari/537.36',
                 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.99 Safari/537.36',
                 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_1) AppleWebKit/602.2.14 (KHTML, like Gecko) Version/10.0.1 Safari/602.2.14',
                 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.71 Safari/537.36',
                 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.98 Safari/537.36',
                 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.98 Safari/537.36',
                 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.71 Safari/537.36',
                 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.99 Safari/537.36',
                 'Mozilla/5.0 (Windows NT 10.0; WOW64; rv:50.0) Gecko/20100101 Firefox/50.0']

def random_headers():
    return {'User-Agent': choice(desktop_agents),'Accept':'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8'}

base_url ='https://allegro.pl/kategoria/laptopy-491'

def get_url():
    for page_number in range(1, 3):

        url = f'{base_url}?p={page_number}'

        response = requests.get(url, headers=random_headers())

        if response.status_code == 200:

            soup = BeautifulSoup(response.text, "lxml")
            data = soup.find_all("div", class_="mpof_ki mp7g_oh mg9e_8 mj7a_8 m7er_k4 mjyo_6x mgmw_3z _6a66d_i2yJ- _6a66d_C1oHB mx7m_1 mnyp_co mlkp_ag mwdn_1 mh36_24 mvrt_24 _6a66d_u7-8J")

            for card in data:
                card_url = "https://allegro.pl" + card.find("a").get("href")
                yield card_url
        else:
            print(f"Failed to load page {page_number}. Status code: {response.status_code}")

Что можно исправить или наоборот еще дописать?

Answer 1

[Михаил Р.]

Обнаружил некую защиту "на лоха", а значит request и обычный selenium не справятся, буду использовать selenium + selenium-stealth (пример от сюда) + chrome + socks5 (рабочий, на момент теста):

import time
from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.chrome.options import Options
from selenium_stealth import stealth

proxy = '45.132.75.19:23820'

options = Options()
options.add_argument('--proxy-server=socks5://' + proxy)
options.add_argument("--disable-blink-features=AutomationControlled")
options.add_argument("window-size=1920,1080")
# options.add_argument("--incognito")
options.add_argument("--disable-extensions")
options.add_argument("--disable-plugins-discovery")
options.add_argument("--start-maximized")
options.add_experimental_option("excludeSwitches", ["enable-automation"])
options.add_experimental_option('useAutomationExtension', False)

executable_path = './chromedriver-121.0.6167.85.exe'

service = Service(executable_path=executable_path)
driver = webdriver.Chrome(service=service, options=options)

stealth(driver,
        languages=["en-US", "en"],
        vendor="Google Inc.",
        platform="Win64",
        webgl_vendor="Intel Inc.",
        renderer="Intel Iris OpenGL Engine",
        fix_hairline=True,
        run_on_insecure_origins=True,
        user_agent='Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.6167.85 Safari/537.36',
        )

try:
    driver.get('https://allegro.pl/kategoria/laptopy-491')
    time.sleep(30)
    print(driver.page_source)
finally:
    driver.quit()

Chromedriver качать здесь, он должен совпадать с мажорной версией Вашего Chrome браузера, который будет использовать Selenium.