Tried reading Request Headers with Browsermob + Selenium in Java but only got empty har

Question

My goal is to read the request headers of the app portal.
Am using browsermob-core with selenium, but the entries in har always comes empty.
I tried removing the headless arg and also tried using a separate SSLProxy for BMP, but still the har is empty. Heres my code:

package com.example;

import io.github.bonigarcia.wdm.WebDriverManager;
import net.lightbody.bmp.BrowserMobProxy;
import net.lightbody.bmp.BrowserMobProxyServer;
import net.lightbody.bmp.client.ClientUtil;
import net.lightbody.bmp.core.har.Har;
import net.lightbody.bmp.core.har.HarEntry;
import net.lightbody.bmp.proxy.CaptureType;
import org.openqa.selenium.By;
import org.openqa.selenium.Proxy;
import org.openqa.selenium.WebDriver;
import org.openqa.selenium.WebElement;
import org.openqa.selenium.chrome.ChromeDriver;
import org.openqa.selenium.chrome.ChromeOptions;
import org.openqa.selenium.remote.CapabilityType;
import org.openqa.selenium.remote.DesiredCapabilities;
import org.openqa.selenium.support.ui.ExpectedConditions;
import org.openqa.selenium.support.ui.WebDriverWait;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.File;
import java.time.Duration;
import java.util.List;
import java.util.concurrent.TimeUnit;

public class webDriver {
    private static final Logger logger
            = LoggerFactory.getLogger(webDriver.class);
            //= LoggerFactory.getLogger(ch.qos.logback.classic.Logger.ROOT_LOGGER_NAME);

    public static void generateUIToken() throws Exception {
        
        WebDriver driver = null;
        BrowserMobProxy proxy = new BrowserMobProxyServer();
        proxy.setTrustAllServers(true);
        proxy.start(0);
        Har har;
        Proxy seleniumProxy = ClientUtil.createSeleniumProxy(proxy);
        String hostIp = "localhost";
        seleniumProxy.setHttpProxy(hostIp + ":" + proxy.getPort());
        seleniumProxy.setSslProxy(hostIp + ":" + proxy.getPort());
        seleniumProxy.setSslProxy("trustAllSSLCertificates");

        WebDriverManager.chromedriver().setup();
        ChromeOptions options = new ChromeOptions();
        options.addArguments("--no-sandbox");
        options.addArguments("--headless");
        options.addArguments("--ignore-certificate-errors");
        options.addArguments("--disable-dev-shm-usage");

        DesiredCapabilities capabilities = new DesiredCapabilities();
        capabilities.setCapability(CapabilityType.PROXY, seleniumProxy);
        capabilities.setCapability(CapabilityType.ACCEPT_INSECURE_CERTS,true);
        capabilities.setCapability(CapabilityType.ACCEPT_SSL_CERTS,true);
        options.merge(capabilities);

        try {

            driver = new ChromeDriver(options);



            proxy.enableHarCaptureTypes(CaptureType.REQUEST_CONTENT, CaptureType.RESPONSE_CONTENT);

            proxy.setHarCaptureTypes(CaptureType.REQUEST_HEADERS, CaptureType.RESPONSE_HEADERS);
            proxy.enableHarCaptureTypes(CaptureType.REQUEST_HEADERS,CaptureType.RESPONSE_HEADERS);
            proxy.enableHarCaptureTypes(CaptureType.REQUEST_CONTENT, CaptureType.RESPONSE_CONTENT);
            proxy.newHar("new.example.com");

            driver.get("https://new.example.com");

            String title = driver.getTitle();
            TimeUnit.SECONDS.sleep(10);
            WebDriverWait wait = new WebDriverWait(driver, Duration.ofSeconds(20));
            wait.until(ExpectedConditions.elementToBeClickable(By.id("userInput")));
            WebElement user_login = driver.findElement(By.id("userInput"));
            user_login.sendKeys("username");
            user_login.submit();
            logger.info(user_login + " - " + title);
            TimeUnit.SECONDS.sleep(10);

            wait.until(ExpectedConditions.elementToBeClickable(By.id("okta-signin-password")));
            WebElement pass_login = driver.findElement(By.id("okta-signin-password"));
            pass_login.sendKeys("password");
            pass_login.submit();
            logger.info(pass_login + " - " + title);

            TimeUnit.SECONDS.sleep(30);

            logger.info("Current" + driver.getCurrentUrl());
            har = proxy.getHar();
            List<HarEntry> entries = har.getLog().getEntries();
            for (HarEntry entry : entries) {
                logger.info("Request URL: " + entry.getRequest().getUrl());
                logger.info("Entry response status: " + entry.getResponse().getStatus());
                logger.info("Entry response text: " + entry.getResponse().getStatusText());

            }
            har.writeTo(new File("test.har"));
            TimeUnit.SECONDS.sleep(10);
            driver.quit();
            proxy.endHar();
            proxy.stop();

        }
        catch (Exception e){
            assert driver != null;
            driver.close();
            driver.quit();
        }
    }

    public static void main(String[] args) throws Exception {

        generateUIToken();


    }
}

And heres the output in har file:

{"log":{"version":"1.2","creator":{"name":"BrowserMob Proxy","version":"2.1.5","comment":""},"pages":[{"id":"new.example.com","startedDateTime":"2022-12-06T21:46:04.043Z","title":"new.example.com","pageTimings":{"comment":""},"comment":""}],"entries":[],"comment":""}}

Am using Java 17 and browsermob-core 2.1.5 with Selenium 4.7.0
Can anyone help me figure out why the har is always having empty entries?

Answer 1

Its exactly not answer your question but your problem might solve in other way. Selenium can intercept.network and get request and response header. In selenium have 4.0.6 have that capablity.

    <dependency>
    
      <groupId>org.seleniumhq.selenium</groupId>
    
      <artifactId>selenium-java</artifactId>
    
      <version>4.6.0</version>

</dependency>

Here is the full code for how to intercept

public static void main(String[] args) throws InterruptedException, AWTException {

    ChromeOptions options = new ChromeOptions();

    ChromeDriver driver = new ChromeDriver(options);
    driver.get("https://www.google.com");
    DevTools devTools = driver.getDevTools();
    devTools.createSession();

    devTools.send(Network.enable(Optional.empty(),Optional.empty(),Optional.empty()));

    devTools.addListener(Network.requestWillBeSent(),
            request ->{
                System.out.println("Request URL:"+request.getRequest().getUrl());
                System.out.println("Request Method:"+request.getRequest().getMethod());
                System.out.println("Request Method:"+request.getRequest().getHeaders().toJson());
            });
    Thread.sleep(50000);
}

Now you can save all request inside a file a JSON or har extension