Python
- 44 ответа
- 0 вопросов
72
Вклад в тег
import requests
from bs4 import BeautifulSoup
import json
load_url = 'https://www.biz-cen.ru/load/'
search_params = {"metro":{"lines":{}},"was_ra":",","limit":20,"to_ra":0,"tolim":20,"bc_in_fav":[],"office_in_fav":[],"bcs_in_view_start":[],"num_in":3,"typeof_search":5,"show_fav":0}
data = {'search_params':json.dumps(search_params), 'was_bc_loaded':0}
session = requests.Session()
seen = set()
def parse(response):
soup = BeautifulSoup(response.text, 'lxml')
table = soup.find('ul', id='bObjDataList')
if table:
lis = table.find_all('li')
else:
lis = soup.find_all('li')
return [i.find('a').get('href') for i in lis]
while len(seen) < 200:
response = session.post(load_url, data=data)
for link in parse(response):
seen.add(link)
search_params['limit'] += 20
data['search_params'] = json.dumps(search_params)
print(len(seen))