from utils.retry import ( retry_func, locate_element_with_retry, find_clickable_element_with_retry, ) from time import sleep import re from bs4 import BeautifulSoup from utils.config import config if config.open_driver: try: from selenium.webdriver.common.by import By except: pass def get_soup_driver(url): """ Get the soup by driver """ from driver.setup import setup_driver driver = setup_driver() retry_func(lambda: driver.get(url), name=url) sleep(1) source = re.sub( r"", "", driver.page_source, flags=re.DOTALL, ) soup = BeautifulSoup(source, "html.parser") driver.close() driver.quit() return soup def search_submit(driver, name): """ Input key word and submit with driver """ search_box = locate_element_with_retry(driver, (By.XPATH, '//input[@type="text"]')) if not search_box: return search_box.clear() search_box.send_keys(name) submit_button = find_clickable_element_with_retry( driver, (By.XPATH, '//input[@type="submit"]') ) if not submit_button: return driver.execute_script("arguments[0].click();", submit_button)