Доброго времени суток
Сделал парсинг сайта :
import requests
from bs4 import BeautifulSoup
from selenium import webdriver
import csv
import os
from threading import *
import requests
from concurrent.futures import ThreadPoolExecutor, wait
import time
firefox_profile = webdriver.FirefoxProfile()
firefox_profile.set_preference("thatoneguydotnet.QuickJava.startupStatus.Images", 2) ## Turns images off
firefox_profile.set_preference("thatoneguydotnet.QuickJava.startupStatus.AnimatedImage", 2) ## Turns animated
URL = 'https://novgorod.shop.megafon.ru/mobile'
HEDARS = {'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.0.3 Safari/605.1.15', 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8'}
FILE = 'meg.csv'
direct= os.getcwd()
def get_himl(url):
driver = (direct + '/geckodriver')
option = webdriver.FirefoxOptions()
option.set_preference('dom.webdriver.enabled', False)
#option.headless = True
browser = webdriver.Firefox(executable_path=driver, options=option)
time.sleep(3)
browser.get(url)
time.sleep(5)
r = browser.page_source
time.sleep(3)
browser.quit()
return r
def seve_file(item,path):
with open(path, 'w', newline='',encoding='utf-8') as file:
writer = csv.writer(file, delimiter = ';')
writer.writerow(['бренд', 'модель', 'цена', 'акции'])
for items in item:
writer.writerow([items['brand'], items['title'], items['prise'], items['sale']])
def multiple_replace(target_str, replace_values):
for i, j in replace_values.items():
target_str = target_str.replace(i, j)
return target_str
def get_content(html):
soup = BeautifulSoup(html, 'html.parser')
items = soup.find_all(class_='b-goods-list__item')
phone = []
for items in items:
prise = items.find('span', class_='b-price-good-list__value b-price__value')
if prise:
prise = prise.get_text()
else:
prise = ''
action = items.find_all('div', class_='c-actionTip__wrapper c-actionTip__text c-actionTip__text--small')
sale = []
if action:
for action in action:
sale += action
sale = (' '.join(map(str,sale)))
replace_values = {'<br/>': '','<p class="c-actionTip__price">':'', '\n':'','<p>':'','</p>':'','VoLTE ':''}
sale = multiple_replace(sale, replace_values)
else:
sale = ''
name = items.find('a', class_='b-good__title-link').get_text()
name = name.split()
name = ' '.join(name[0:-1]).lower()
brand = name.split()[0]
phone.append({
'brand' : brand,
'title' : name,
'prise' : prise,
'sale' : sale
})
return (phone)
def pars():
for i in range(1,2):
print(f'анаизруем {i} страницу ')
a = (URL + '?page=' + str(i) + '&archVal=0')
html = get_himl(a)
phone.extend(get_content(html))
seve_file(phone, FILE)
print(f'найдено ' + str(len(phone)) + ' телефонов')
if __name__ == "__main__":
phone = []
pars()
Но при открытии страницы вылетает сообщение:
Access Denied
You don't have permission to access "/mobile?page=1&archVal=0" on this server.
Reference ID: 6d3f093e-e839-421f-b93e-94e3419d86e7
Description: 42
Отказ в доступе
Кто ни будь может подсказать как обойти защиту?
