How can I scroll a particular section of a dynamic web page using selenium webdriver in python?

Question

I have found many reference that scroll the entire webpage but I am looking for a particular section to scroll. I am working on marketwatch.com - section - latest news tab. How can I scroll just this latest news tab using selenium webdriver?

Below is my code which returns the heading of the news but keeps repeating same headings.

from bs4 import BeautifulSoup
import urllib
import csv
import time
from selenium import webdriver


count = 0   
browser = webdriver.Chrome()
browser.get("https://www.marketwatch.com/newsviewer")

pageSource = browser.page_source

soup = BeautifulSoup(pageSource, 'lxml')

arkodiv = soup.find("ol", class_="viewport")

while browser.find_element_by_tag_name('ol'):
    browser.execute_script("window.scrollTo(0, document.body.scrollHeight);")
    time.sleep(0.5)
    div = list(arkodiv.find_all('div', class_= "nv-details"))

    heading = []
    Data_11 = list(soup.find_all("div", class_ = "nv-text-cont"))          

    datetime = list(arkodiv.find_all("li", timestamp = True))
    for sa in datetime:
        sh = sa.find("div", class_ = "nv-text-cont")
        if sh.find("a", class_ = True):
            di = sh.text.strip()
            di = di.encode('ascii', 'ignore').decode('ascii')
        else:
            continue
        print di
        heading.append((di))       
        count = count+1         


    if 'End of Results' in arkodiv:
        print 'end'
        break
    else:
        continue
    print count

Answer 1

That happens because the script you are executing scrolls to the bottom of the page.

To keep scrolling inside the element fetching news you need to replace this:

browser.execute_script("window.scrollTo(0, document.body.scrollHeight);")

with this:

browser.execute_script("document.documentElement.getElementsByClassName('viewport')[0].scrollTop = 999999")

EDIT

This is the complete working solution:

from bs4 import BeautifulSoup
import urllib
import csv
import time
from selenium import webdriver


count = 0   
browser = webdriver.Chrome()
browser.get("https://www.marketwatch.com/newsviewer")

while browser.find_element_by_tag_name('ol'):

    pageSource = browser.page_source
    soup = BeautifulSoup(pageSource, 'lxml')
    arkodiv = soup.find("ol", class_="viewport")
    browser.execute_script("document.documentElement.getElementsByClassName('viewport')[0].scrollTop = 999999")
    time.sleep(0.5)
    div = list(arkodiv.find_all('div', class_= "nv-details"))

    heading = set()
    Data_11 = list(soup.find_all("div", class_ = "nv-text-cont"))          

    datetime = list(arkodiv.find_all("li", timestamp = True))
    for sa in datetime:
        sh = sa.find("div", class_ = "nv-text-cont")
        if sh.find("a", class_ = True):
            di = sh.text.strip()
            di = di.encode('ascii', 'ignore').decode('ascii')
        else:
            continue
        print di
        heading.add((di))       
        count = count+1         


    if 'End of Results' in arkodiv:
        print 'end'
        break
    else:
        continue
    print count

EDIT 2

You may also want to change how you store the headers, since the way you currently do keeps duplicates inside the list. Changed it to a set so that doesn't happen.