Selenium Webdriver with Firebug + NetExport + FireStarter not creating a har file in Python

Question

I am currently running Selenium with Firebug, NetExport, and (trying out) FireStarter in Python trying to get the network traffic of a URL. I expect a HAR file to appear in the directory listed, however nothing appears. When I test it in Firefox and go through the UI, a HAR file is exported and saved so I know the code itself functions as expected. After viewing multiple examples I do not see what I am missing.

I am using Firefox 29.0.1 Firebug 1.12.8 FireStarter 0.1a6 NetExport 0.9b6

Has anyone else encountered this issue? I am receiving a "webFile.txt" file being filled out correctly.

After looking up each version of the add-ons they are supposed to be compatible with the version of Firefox I am using. I tried using Firefox version 20, however that did not help. I am currently pulling source code.

In addition I have tried it with and without FireStarter, and I have tried refreshing the page manually in both cases to try to generate a HAR.

My code looks like this:

import urllib2
import sys
import re
import os
import subprocess
import hashlib
import time
import datetime
from browsermobproxy import Server
from selenium import webdriver
import selenium
a=[];
theURL='';
fireBugPath = '/Users/tai/Documents/workspace/testSelenium/testS/firebug.xpi';
netExportPath = '/Users/tai/Documents/workspace/testSelenium/testS/netExport.xpi';
fireStarterPath = '/Users/tai/Documents/workspace/testSelenium/testS/fireStarter.xpi';

profile = webdriver.firefox.firefox_profile.FirefoxProfile();
profile.add_extension( fireBugPath);
profile.add_extension(netExportPath);
profile.add_extension(fireStarterPath);

#firefox preferences
profile.set_preference("app.update.enabled", False)  
profile.native_events_enabled = True
profile.set_preference("webdriver.log.file", "/Users/tai/Documents/workspace/testSelenium/testS/webFile.txt")
profile.set_preference("extensions.firebug.DBG_STARTER", True);

profile.set_preference("extensions.firebug.currentVersion", "1.12.8");
profile.set_preference("extensions.firebug.addonBarOpened", True);
profile.set_preference("extensions.firebug.addonBarOpened", True);
profile.set_preference('extensions.firebug.consoles.enableSite', True)                          


profile.set_preference("extensions.firebug.console.enableSites", True);
profile.set_preference("extensions.firebug.script.enableSites", True);
profile.set_preference("extensions.firebug.net.enableSites", True);
profile.set_preference("extensions.firebug.previousPlacement", 1);
profile.set_preference("extensions.firebug.allPagesActivation", "on");
profile.set_preference("extensions.firebug.onByDefault", True);
profile.set_preference("extensions.firebug.defaultPanelName", "net");

#set net export preferences
profile.set_preference("extensions.firebug.netexport.alwaysEnableAutoExport", True);
profile.set_preference("extensions.firebug.netexport.autoExportToFile", True);
profile.set_preference("extensions.firebug.netexport.saveFiles", True);

profile.set_preference("extensions.firebug.netexport.autoExportToServer", False);
profile.set_preference("extensions.firebug.netexport.Automation", True);
profile.set_preference("extensions.firebug.netexport.showPreview", False);
profile.set_preference("extensions.firebug.netexport.pageLoadedTimeout", 15000);
profile.set_preference("extensions.firebug.netexport.timeout", 10000);

profile.set_preference("extensions.firebug.netexport.defaultLogDir", "/Users/tai/Documents/workspace/testSelenium/testS/har");
profile.update_preferences();
browser = webdriver.Firefox(firefox_profile=profile);


def openURL(url,s):
    theURL = url;
    time.sleep(6);
    #browser = webdriver.Chrome();
    browser.get(url); #load the url in firefox
    time.sleep(3); #wait for the page to load
    browser.execute_script("window.scrollTo(0, document.body.scrollHeight/5);")
    time.sleep(1); #wait for the page to load
    browser.execute_script("window.scrollTo(0, document.body.scrollHeight/4);")
    time.sleep(1); #wait for the page to load
    browser.execute_script("window.scrollTo(0, document.body.scrollHeight/3);")
    time.sleep(1); #wait for the page to load
    browser.execute_script("window.scrollTo(0, document.body.scrollHeight/2);")
    time.sleep(1); #wait for the page to load
    browser.execute_script("window.scrollTo(0, document.body.scrollHeight);")
    searchText='';
    time.sleep(20); #wait for the page to load
    if(s.__len__() >0):
        for x in range(0, s.__len__()):
            searchText+= (""  + browser.find_element_by_id(x));
    else:
        searchText+= browser.page_source;

        a=getMatches(searchText)
    #print ("\n".join(swfLinks));
    print('\n'.join(removeNonURL(a)));
   # print(browser.page_source);
    browser.quit();
    return a;
def found_window(name):
        try: browser.switch_to_window(name)
        except NoSuchWindowException:
             return False
        else:
             return True # found window
def removeFirstQuote(tex):
    for x in tex:
        b = x[1:];
        if not b in a:
            a.append(b);
    return a;
def getMatches(t):
    return removeFirstQuote(re.findall('([\"|\'][^\"|\']*\.swf)', t));
def removeNonURL(t):
    a=[];
    for b in t:
        if(b.lower()[:4] !="http" ):
            if(b[0] == "//"):
                a.append(theURL+b[2:b.__len__()]);
            else:
                while(b.lower()[:4] !="http" and b.__len__() >5):
                    b=b[1:b.__len__()];
                a.append(b);
        else:
            a.append(b);
    return a;

openURL("http://www.chron.com",a);

Answer 1

I fixed this issue for my own work by setting a longer wait before closing the browser. I think you are currently setting netexport to export after the program has quit, so no file is written. The line causing this is:

profile.set_preference("extensions.firebug.netexport.pageLoadedTimeout", 15000);

From the netexport source code we have that pageLoadedTimeout is the `Number of milliseconds to wait after the last page request to declare the page loaded'. So I suspect all your minor page loads are preventing netexport from having enough time to write the file. One caveat is you set the system to automatically export after 10s so I'm not sure why you are not acquiring half loaded json files.