Выскакивает ошибка 'NoneType' object has no attribute 'select', как её исправить?

Question

[N T]

Написал код, который должен выводить в json информацию с конкурсов сайта, включающую в себя: название, описание конкурса, начало и окончание приёма заявок, географию конкурса, организатора конкурса и т.д. Примерно так должно выглядеть:

[
 {
  "Название": "",
  "Описание": "",
  "Старт приема заявок": "",
  "Окончание приема заявок": "",
  "Участники конкурса": ""  
  "География конкурса": ""  
  "Организатор конкурса": ""  
 }
]

Сперва я попробовал написать код, который выведет в json название того или иного конкурса:

import requests
from bs4 import BeautifulSoup
import re
import os
from urllib.request import urlopen
import json
from urllib.parse import unquote
import asposecells
import jpype
from openpyxl import Workbook
import pandas as pd
import mysql.connector
from seleniumwire import webdriver 
import warnings
warnings.filterwarnings("ignore")
import time
import cloudscraper
from selenium import webdriver

url = 'https://гранты.рф/data/grants/list'
OUT_FILENAME = 'out.json'
options = webdriver.EdgeOptions()
options.add_argument("--headless")
driver = webdriver.Edge(options=options)
driver.get(url)
time.sleep(1.5)
html_parsed = driver.page_source
urls = []
soup = BeautifulSoup(html_parsed, 'html.parser')
def get_soup(url, **kwargs):
    response = requests.get(url, **kwargs, verify=False)
    if response.status_code == 200:
        soup = BeautifulSoup(response.text, features='html.parser')
    else:
        soup = None
    return soup

def crawl_products(url):
    for tag in soup.select('.competition-card-wrapper'):
        t = tag.attrs['href']
        url = 'https://гранты.рф{}'.format(t)
        urls.append(url)
        print(urls)
    return urls

def parse_products(urls):
    data = []
    item = {}
    soup = get_soup(url)
    for tr in soup.select(f".grants-competition-page.full-page-competition"):
        name = tr.select_one("p").text
        return data
    item['Название'] = name
    data.append(item)
    
def dump_to_json(filename, data, **kwargs):
    kwargs.setdefault('ensure_ascii', False)
    kwargs.setdefault('indent', 1)
 
    with open(OUT_FILENAME, 'w', encoding="utf-8") as f:
        json.dump(data, f, **kwargs)

def main():
    urls = crawl_products(url)
    data = parse_products(urls)
    dump_to_json(OUT_FILENAME, data)
 
    with open(OUT_FILENAME, 'w', encoding="utf-8") as f:
        json.dump(data, f, ensure_ascii=False, indent=1)

if __name__ == '__main__':
    main()
    
df = pd.read_json('./out.json')

И тут же возникает ошибка. Пробовал подставить любые классы из div, тщетно.

Traceback (most recent call last):
  File "C:\Download\server\py\main2.py", line 72, in <module>
    main()
  File "C:\Download\server\py\main2.py", line 65, in main
    data = parse_products(urls)
  File "C:\Download\server\py\main2.py", line 50, in parse_products
    for tr in soup.select(f".grants-competition-page.full-page-competition"):
AttributeError: 'NoneType' object has no attribute 'select'

Пробовал в data = parse_products(urls) вместо urls поставить что-то другое. Ошибок не было, но парсер пуст.

Answer 1

[N T]

Ошибку исправил. Но теперь парсер не выводится. Пишу код.

import requests
from bs4 import BeautifulSoup
import re
import os
from urllib.request import urlopen
import json
from urllib.parse import unquote
import asposecells
import jpype
from openpyxl import Workbook
import pandas as pd
import mysql.connector
from seleniumwire import webdriver 
import warnings
warnings.filterwarnings("ignore")
import time
import cloudscraper
from selenium import webdriver

url = 'https://гранты.рф/data/grants/list'
OUT_FILENAME = 'out.json'
options = webdriver.EdgeOptions()
options.add_argument("--headless")
driver = webdriver.Edge(options=options)
driver.get(url)
time.sleep(1.5)
html_parsed = driver.page_source
urls = []
soup = BeautifulSoup(html_parsed, 'html.parser')
def get_soup(url, **kwargs):
    response = requests.get(url, **kwargs, verify=False)
    if response.status_code == 200:
        soup = BeautifulSoup(response.text, features='html.parser')
    else:
        soup = None
    return soup

def crawl_products(url):
    for tag in soup.select('.competition-card-wrapper'):
        t = tag.attrs['href']
        url = 'https://гранты.рф{}'.format(t)
        urls.append(url)
        print(urls)
    return urls

def parse_products(urls):
    data = []
    item = {}
    for page in urls:
        soup = get_soup(page)
        if soup is None:
            break
        for tr in soup.select(".grants-competition-page-hero__title"):
            name = tr.select_one("p").text
            item['Название'] = name
            data.append(item)
        return data
    
def dump_to_json(filename, data, **kwargs):
    kwargs.setdefault('ensure_ascii', False)
    kwargs.setdefault('indent', 1)
 
    with open(OUT_FILENAME, 'w', encoding="utf-8") as f:
        json.dump(data, f, **kwargs)

def main():
    urls = crawl_products(url)
    data = parse_products(urls)
    dump_to_json(OUT_FILENAME, data)
 
    with open(OUT_FILENAME, 'w', encoding="utf-8") as f:
        json.dump(data, f, ensure_ascii=False, indent=1)

if __name__ == '__main__':
    main()
    
df = pd.read_json('./out.json')