I am trying to extract data from a webpage (https://clinicaltrials.gov), I have built a scraper using selenium and lxml and it is working fine. I need to hit the next page button once the first page scraping is done, after going to the next page I need to take the url of that page using (driver.current_url) and start the scraping again.
Here the problem is search results table only changing but the URL remaining same. So whenever driver taking current url (driver.current_url) first page results coming again and again.
Edited: here is the code
import re
import time
import urllib.parse
import lxml.html
import pandas as pd
import requests
import urllib3
from lxml import etree
from lxml import html
from pandas import ExcelFile
from pandas import ExcelWriter
from selenium import webdriver
from selenium.webdriver.common.action_chains import ActionChains
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.support.wait import WebDriverWait
import selenium.webdriver.support.expected_conditions as EC
siteurl = 'https://clinicaltrials.gov/'
driver = webdriver.Chrome()
driver.get(siteurl)
WebDriverWait(driver, 5)
driver.maximize_window()
def advancesearch():
driver.find_element_by_link_text('Advanced Search').click()
driver.find_element_by_id('StartDateStart').send_keys('01/01/2016')
driver.find_element_by_id('StartDateEnd').send_keys('12/30/2020')
webdriver.ActionChains(driver).send_keys(Keys.ENTER).perform()
time.sleep(3)
driver.find_element_by_xpath("//input[contains(@id, 'home-search-condition-query')]").send_keys('medicine') #Give keyword here
advancesearch()
#driver.find_element_by_xpath("//div[contains(@class, 'dataTables_length')]//label//select//option[4]").click()
#time.sleep(8)
def nextbutton():
driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")
time.sleep(5)
driver.find_element_by_xpath("//a[contains(@class, 'paginate_button next')]").click()
def extractor():
cur_url = driver.current_url
read_url = requests.get(cur_url)
souptree = html.fromstring(read_url.content)
tburl = souptree.xpath("//table[contains(@id, 'theDataTable')]//tbody//tr//td[4]//a//@href")
for tbu in tburl:
allurl = []
allurl.append(urllib.parse.urljoin(siteurl, tbu))
for tb in allurl:
get_url = requests.get(tb)
get_soup = html.fromstring(get_url.content)
pattern = re.compile("^\s+|\s*,\s*|\s+$")
name = get_soup.xpath('//td[@headers="contactName"]//text()')
phone = get_soup.xpath('//td[@headers="contactPhone"]//text()')
mail = get_soup.xpath('//td[@headers="contactEmail"]//a//text()')
artitle = get_soup.xpath('//td[@headers="contactEmail"]//a//@href')
artit = ([x for x in pattern.split(str(artitle)) if x][-1])
title = artit[:-2]
for (names, phones, mails) in zip(name, phone, mail):
fullname = names[9:]
print(fullname, phones, mails, title, sep='\t')
while True:
extractor()
nextbutton()