Pull variable value from javascript source using BeautifulSoup4 Python

Question

I'm newbie in python programming. I'm learning beautifulsoup to scrap website.

I want to extract and store the value of "stream" to my variable.

My Python code as follows :

import bs4 as bs #Importing BeautifulSoup4 Python Library.
import urllib.request
import requests
import json
import re

headers = {'User-Agent':'Mozilla/5.0'}
url = "http://thoptv.com/partners/mhdTVlive/Core.php?level=1200&channel=Dsports_HD"

page = requests.get(url)
soup = bs.BeautifulSoup(page.text,"html.parser")
pattern = re.compile('var stream = (.*?);')
scripts = soup.find_all('script')

for script in scripts:
   if(pattern.match(str(script.string))):
       data = pattern.match(script.string)
       links = json.loads(data.groups()[0])
       print(links)

This is the source javascript code to get the stream url value.

https://content.jwplatform.com/libraries/oncyToRO.js'>if( navigator.userAgent.match(/android/i)|| navigator.userAgent.match(/webOS/i)|| navigator.userAgent.match(/iPhone/i)|| navigator.userAgent.match(/iPad/i)|| navigator.userAgent.match(/iPod/i)|| navigator.userAgent.match(/BlackBerry/i)|| navigator.userAgent.match(/Windows Phone/i)) {var stream = " http://ssrigcdnems01.cdnsrv.jio.com/jiotv.live.cdn.jio.com/Dsports_HD/Dsports_HD_800.m3u8?jct=ibxIPxc6rkq1yIUJb4RlEV&pxe=1504146411&st=AQIC5wM2LY4SfczRaEwgGl4Dyvly_3HihdlD_Oduojk5Kxs . AAJTSQACMDIAAlNLABQtNjUxNDEwODczODgxNzkyMzg5OQACUzEAAjYw ";}else{var stream = " http://hd.simiptv.com:8080//index.m3u8?key=VIoVSsGRLRouHWGNo1epzX&exp=932213423&domain=thoptv.stream&id=461 ";}jwplayer("THOPTVPlayer").setup({"title": 'thoptv.stream',"stretching":"exactfit","width": "100%","file": none,"height": "100%","skin": "seven","autostart": "true","logo": {"file":" https://i.imgur.com/EprI2uu.png ","margin":"-0", "position":"top-left","hide":"false","lin k":" http://mhdtvlive.co.in "},"androidhls": true,});jwplayer("THOPTVPlayer").onError(function(){jwplayer().load({file:" http://content.jwplatform.com/videos/7RtXk3vl-52qL9xLP.mp4 ",image:" http://content.jwplatform.com/thumbs/7RtXk3vl-480.jpg "});jwplayer().play();});jwplayer("THOPTVPlayer").onComplete(function(){window.location = window.location.href;});jwplayer("THOPTVPlayer").onPlay(function(){clearTimeout(theTimeout);});

I need to extract the url from stream.

var stream = " http://ssrigcdnems01.cdnsrv.jio.com/jiotv.live.cdn.jio.com/Dsports_HD/Dsports_HD_800.m3u8?jct=ibxIPxc6rkq1yIUJb4RlEV&pxe=1504146411&st=AQIC5wM2LY4SfczRaEwgGl4Dyvly_3HihdlD_Oduojk5Kxs . AAJTSQACMDIAAlNLABQtNjUxNDEwODczODgxNzkyMzg5OQACUzEAAjYw ";}

Answer 1

Rather then thinking complicated with regex, if the link is the only dynamically changing part, you can split the string with some known separating tokens.

x = """
https://content.jwplatform.com/libraries/oncyToRO.js'>if( navigator.userAgent.match(/android/i)|| navigator.userAgent.match(/webOS/i)|| navigator.userAgent.match(/iPhone/i)|| navigator.userAgent.match(/iPad/i)|| navigator.userAgent.match(/iPod/i)|| navigator.userAgent.match(/BlackBerry/i)|| navigator.userAgent.match(/Windows Phone/i)) {var stream = "http://ssrigcdnems01.cdnsrv.jio.com/jiotv.live.cdn.jio.com/Dsports_HD/Dsports_HD_800.m3u8?jct=ibxIPxc6rkq1yIUJb4RlEV&pxe=1504146411&st=AQIC5wM2LY4SfczRaEwgGl4Dyvly_3HihdlD_Oduojk5Kxs.AAJTSQACMDIAAlNLABQtNjUxNDEwODczODgxNzkyMzg5OQACUzEAAjYw";}else{var stream = "http://hd.simiptv.com:8080//index.m3u8?key=VIoVSsGRLRouHWGNo1epzX&exp=932213423&domain=thoptv.stream&id=461";}jwplayer("THOPTVPlayer").setup({"title": 'thoptv.stream',"stretching":"exactfit","width": "100%","file": none,"height": "100%","skin": "seven","autostart": "true","logo": {"file":"https://i.imgur.com/EprI2uu.png","margin":"-0", "position":"top-left","hide":"false","link":"http://mhdtvlive.co.in"},"androidhls": true,});jwplayer("THOPTVPlayer").onError(function(){jwplayer().load({file:"http://content.jwplatform.com/videos/7RtXk3vl-52qL9xLP.mp4",image:"http://content.jwplatform.com/thumbs/7RtXk3vl-480.jpg"});jwplayer().play();});jwplayer("THOPTVPlayer").onComplete(function(){window.location = window.location.href;});jwplayer("THOPTVPlayer").onPlay(function(){clearTimeout(theTimeout);});
"""
left1, right1 = x.split("Phone/i)) {var stream =")
left2, right2 = right1.split(";}else")

print(left2)
# "http://ssrigcdnems01.cdnsrv.jio.com/jiotv.live.cdn.jio.com/Dsports_HD/Dsports_HD_800.m3u8?jct=ibxIPxc6rkq1yIUJb4RlEV&pxe=1504146411&st=AQIC5wM2LY4SfczRaEwgGl4Dyvly_3HihdlD_Oduojk5Kxs.AAJTSQACMDIAAlNLABQtNjUxNDEwODczODgxNzkyMzg5OQACUzEAAjYw"

Answer 2

pattern.match() matches the pattern from the beginning of the string. Try using pattern.search() instead - it will match anywhere within the string.

Change your for loop to this:

for script in scripts:
    data = pattern.search(script.text)
    if data is not None:
        stream_url = data.groups()[0]
        print(stream_url)

You can also get rid of the surrounding quotes by changing the regex pattern to:

pattern = re.compile('var stream = "(.*?)";')

so that the double quotes are not included in the group.

---

You might also have noticed that there are two possible stream variables depending on the accessing user agent. For tablet like devices the first would be appropriate, while all other user agents should use the second stream . You can use pattern.findall() to get all of them:

>>> pattern.findall(script.text)
['"http://ssrigcdnems01.cdnsrv.jio.com/jiotv.live.cdn.jio.com/Dsports_HD/Dsports_HD_800.m3u8?jct=LEurobVVelOhbzOZ6EkTwr&pxe=1571716053&st=AQIC5wM2LY4SfczRaEwgGl4Dyvly_3HihdlD_Oduojk5Kxs.*AAJTSQACMDIAAlNLABQtNjUxNDEwODczODgxNzkyMzg5OQACUzEAAjYw*"', '"http://hd.simiptv.com:8080//index.m3u8?key=vaERnLJswnWXM8THmfvDq5&exp=944825312&domain=thoptv.stream&id=461"']

Answer 3

this code works for me

import bs4 as bs #Importing BeautifulSoup4 Python Library.
import urllib.request
import requests
import json


headers = {'User-Agent':'Mozilla/5.0'}
url = "http://thoptv.com/partners/mhdTVlive/Core.php?
level=1200&channel=Dsports_HD"

page = requests.get(url)
soup = bs.BeautifulSoup(page.text,"html.parser")

scripts = soup.find_all('script')



out = list()
for c, i in enumerate(scripts): #go over list
    text = i.text
    if(text[:2] == "if"): #if the (if) comes first 
        for count, t in enumerate(text): # then we have reached the correct item in the list
            if text[count] == "{" and text[count + 1] == "v" and text[count + 5] == "s": # and if this is here that stream is set
                tmp = text[count:] # add this to the tmp varible
                break # and end
co = 0
for m in tmp: #loop over the results from prev. result
    if m == "\"" and co == 0: #if string is starting
        co = 1 #set count to "true" 1
    elif m == "\"" and co == 1: # if it is ending stop
        print(''.join(out)) #results
        break
    elif co == 1:
        # as long as we are looping over the rigth string
        out.append(m) #add to out list
        pass

result = ''.join(out) #set result

it basicly filters the string manuely.

but if we use user1767754 method (brilliant by the way) we will end up something like this:

import bs4 as bs #Importing BeautifulSoup4 Python Library.
import urllib.request
import requests
import json

headers = {'User-Agent':'Mozilla/5.0'}
url = "http://thoptv.com/partners/mhdTVlive/Core.php?level=1200&channel=Dsports_HD"

page = requests.get(url)
soup = bs.BeautifulSoup(page.text,"html.parser")

scripts = soup.find_all('script')

x = scripts[3].text

left1, right1 = x.split("Phone/i)) {var stream =")
left2, right2 = right1.split(";}else")

print(left2)