from selenium import webdriver
from selenium.webdriver.chrome.options import Options
import time
f = open('text.txt', 'a', encoding='utf8')
urls = open("input.txt", "r")
for url in urls:
def get_url(driver):
driver.get(url)
print("GOT URL")
time.sleep(3)
def press_cookie_btn(driver):
cookie_btn = driver.find_element_by_xpath("//div[@class='topinfo rel']"
"/button[@class='cookie-close abs cookiesBarClose']")
cookie_btn.click()
print("COOKIE")
time.sleep(2)
def get_content(driver):
try:
time.sleep(1)
driver.find_element_by_xpath("//span[@class='link spoiler small nowrap']/span").click()
time.sleep(2)
try:
phone = driver.find_element_by_xpath("//strong[@class='fnormal xx-large']").text
print(phone)
f.write(phone + '\n')
time.sleep(1)
except:
phone_1 = driver.find_element_by_xpath("//strong[@class='fnormal xx-large']/span[@class='block'][1]").text
phone_2 = driver.find_element_by_xpath("//strong[@class='fnormal xx-large']/span[@class='block'][2]").text
print(phone_1, phone_2)
f.write(phone_1 + ' ' + phone_2 + '\n')
time.sleep(1)
except:
pass
def page_pagination(driver):
ars = driver.find_elements_by_xpath("//a[@class='marginright5 link linkWithHash detailsLink']")
urls_1 = []
for ar in ars:
url_1 = ar.get_attribute("href")
urls_1.append(url_1)
print(url_1)
for url_2 in urls_1:
driver.get(url_2)
time.sleep(3)
get_content(driver)
time.sleep(3)
def pages_pagination(driver, last_elem):
page_pagination(driver)
try:
for i in range(1, last_elem):
driver.get(url+"/?page="+str(i))
page_pagination(driver)
except:
pass
def main():
options = Options()
options.add_argument('user-agent=Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.67 Safari/537.36')
options.add_extension("D:\\UU\\cjpalhdlnbpafiamejdnhcphjbkeiagm.crx")
driver = webdriver.Chrome(options=options)
driver.implicitly_wait(10)
get_url(driver)
last_elem = driver.find_element_by_xpath("//span[@class='item fleft'][last()]")
press_cookie_btn(driver)
pages_pagination(driver, last_elem)
driver.quit()
main()
urls.close()
f.close()