Python Selenium - Data not visible; no error

Question

I am trying to scrape data from a website using Selenium, I am able to send values but not receiving the result to start scraping. The program is also not throwing any errors. Here's the reproducible code:

# Setting up driver
driver = webdriver.Chrome(executable_path='D:\Program Files\ChromeDriver\chromedriver.exe')

# Opening up the webpage
driver.get("https://www1.nseindia.com/products/content/derivatives/equities/historical_fo.htm")

# Setting the value of Select Instrument
driver.find_element_by_id('instrumentType').send_keys('Index Options')

# Setting the value of Select Symbol
driver.find_element_by_id('symbol').send_keys('NIFTY 50')

# Setting the value of Select Year
driver.find_element_by_id('year').send_keys('2019')

# Setting the value of Select Expiry
select = Select(driver.find_element_by_id('expiryDate'))
noOfExpiries = 2
select.select_by_index(noOfExpiries)

# Setting the value of Select Option Type
cycle = 'PE'
driver.find_element_by_id('optionType').send_keys(cycle)

# Clicking the date range radio button
driver.find_element_by_xpath("//input[@id='rdDateToDate']").click()

# Setting the date range
fromDate = (datetime.strptime(select.options[noOfExpiries].text, '%d-%m-%Y') - timedelta(days=45)).strftime("%d-%b-%Y")
driver.find_element_by_id('fromDate').send_keys(fromDate)

toDate = datetime.strptime(select.options[noOfExpiries].text, '%d-%m-%Y').strftime("%d-%b-%Y")
driver.find_element_by_id('toDate').send_keys(toDate)

print(fromDate, toDate)

# Clicking the Get Data button
driver.find_element_by_id('getButton').click()

Any clue as to what am I missing here?

I receive no element to scrape data from after clicking "Get Data" — meW
– meW, Commented Jun 7, 2021 at 19:07
I see no delays inside the code. Did you define implicit wait timeout? — Prophet
– Prophet, Commented Jun 7, 2021 at 19:25
You can get the content from that site using requests module. Did you try that? — SIM
– SIM, Commented Jun 7, 2021 at 19:37

SIM · Accepted Answer · 2021-06-07 20:52:25Z

Okay, this should do it. I came up with selenium based solution only because you expressed disinterest in requests module:

from selenium import webdriver
from datetime import datetime, timedelta
from selenium.webdriver.common.by import By
from selenium.webdriver import ChromeOptions
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.support.ui import Select
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC

options = ChromeOptions()
options.add_argument('disable-blink-features=AutomationControlled')

with webdriver.Chrome(options=options) as driver:
    wait = WebDriverWait(driver,10)

    driver.get("https://www1.nseindia.com/products/content/derivatives/equities/historical_fo.htm")
    wait.until(EC.presence_of_element_located((By.ID, "instrumentType"))).send_keys('Index Options')
    wait.until(EC.presence_of_element_located((By.ID, "symbol"))).send_keys('NIFTY 50')
    wait.until(EC.presence_of_element_located((By.ID, "year"))).send_keys('2019')
    select = Select(wait.until(EC.presence_of_element_located((By.ID, "expiryDate"))))
    noOfExpiries = 13
    select.select_by_index(noOfExpiries)
    cycle = 'PE'

    wait.until(EC.presence_of_element_located((By.ID, "optionType"))).send_keys(cycle)
    item = wait.until(EC.presence_of_element_located((By.XPATH, "//input[@id='rdDateToDate']")))
    driver.execute_script("arguments[0].click();",item)
    fromDate = (datetime.strptime(select.options[noOfExpiries].text, '%d-%m-%Y') - timedelta(days=45)).strftime("%d-%b-%Y")
    wait.until(EC.presence_of_element_located((By.ID, "fromDate"))).send_keys(fromDate)

    toDate = datetime.strptime(select.options[noOfExpiries].text, '%d-%m-%Y').strftime("%d-%b-%Y")
    wait.until(EC.presence_of_element_located((By.ID, "toDate"))).send_keys(toDate,Keys.ENTER)

    print(fromDate, toDate)
    search_button = wait.until(EC.presence_of_element_located((By.ID, "getButton")))
    driver.execute_script("arguments[0].click();",search_button)
    tabular_content = wait.until(EC.presence_of_element_located((By.CSS_SELECTOR, "#historicalData > table"))).get_attribute("innerHTML")
    print(tabular_content)

Collectives™ on Stack Overflow

Python Selenium - Data not visible; no error

1 Answer 1

1 Comment

Your Answer

Hot Network Questions

Collectives™ on Stack Overflow

1 Answer 1

1 Comment

Your Answer

Sign up or log in

Post as a guest

Related