Пишу скрипт для себя. Сначала все было нормально, данные парсинга добавлялись в БД. Но потом пошли ошибки (на скрине). Где может крыться проблема?
from multiprocessing import connection
import requests
import sqlite3
from datetime import datetime
from brido import check_database
def get_offer(item):
offer = {}
offer["url"] = item["fullUrl"]
offer["offer_id"] = item["id"]
timestamp = datetime.fromtimestamp(item["addedTimestamp"])
timestamp = datetime.strftime(timestamp, '%Y-%M-%D %H:%M:%S')
offer["date"] = timestamp
offer["price"] = item["bargainTerms"]["priceRur"]
offer["address"] = item["geo"]["userInput"]
offer["area"] = item["totalArea"]
offer["rooms"] = item["roomsCount"]
offer["floor"] = item["floorNumber"]
offer["total_floor"] = item["building"]["floorsCount"]
return offer
def get_offers(data):
for item in data["data"]["offersSerialized"]:
offer = get_offer(item)
check_database(offer)
#break
def get_json():
headers = {
'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.110 Safari/537.36',
}
data = '{"jsonQuery":{"region":{"type":"terms","value":[4778]},"_type":"flatrent","room":{"type":"terms","value":[1,2,3,4,5,6,9,7]},"engine_version":{"type":"term","value":2},"for_day":{"type":"term","value":"!1"},"is_by_homeowner":{"type":"term","value":true},"sort":{"type":"term","value":"creation_date_desc"}}}'
response = requests.post('https://api.cian.ru/search-offers/v2/search-offers-desktop/', headers=headers, data=data)
result = response.json()
return result
def main():
data = get_json()
get_offers(data)
headers = {
'authority': 'api.cian.ru',
'accept': '*/*',
'accept-language': 'en-US,en;q=0.9,ru;q=0.8,ru-RU;q=0.7,de;q=0.6,nl;q=0.5,fr;q=0.4',
'content-type': 'text/plain;charset=UTF-8',
'origin': 'https://samara.cian.ru',
'referer': 'https://samara.cian.ru/',
'sec-ch-ua': '" Not A;Brand";v="99", "Chromium";v="101", "Google Chrome";v="101"',
'sec-ch-ua-mobile': '?0',
'sec-ch-ua-platform': '"Windows"',
'sec-fetch-dest': 'empty',
'sec-fetch-mode': 'cors',
'sec-fetch-site': 'same-site',
'user-agent': 'Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.4951.67 Safari/537.36',
}
data = '{"jsonQuery":{"commission_type":{"type":"term","value":0},"_type":"flatrent","room":{"type":"terms","value":[1,2,7,9]},"for_day":{"type":"term","value":"!1"},"price":{"type":"range","value":{"lte":20000}},"engine_version":{"type":"term","value":2},"currency":{"type":"term","value":2},"with_neighbors":{"type":"term","value":false},"publish_period":{"type":"term","value":-2},"region":{"type":"terms","value":[4768]}}}'
response = requests.post('https://api.cian.ru/search-offers/v2/search-offers-desktop/', headers=headers, data=data)
data = response.json()
#with open('data.json', 'w', encoding='utf-8') as file:
#json.dump(data, file, ensure_ascii=False)
if __name__ == '__main__':
main()
