How to export metrics (collected by DropWizard) in Prometheus format from java application?

Question

My aim - create spring boot application, collect metrics using DropWizard and expose endpoint for Prometheus to consume application metrics:

My code:

@SpringBootApplication
@EnableMetrics(proxyTargetClass = true)
public class DemoApplication {

    public static void main(String[] args) {

        SpringApplication.run(DemoApplication.class, args);
    }

}

---

package com.example.demo;

import com.codahale.metrics.Counter;
import com.codahale.metrics.MetricRegistry;
import com.codahale.metrics.annotation.Timed;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RestController;

import javax.annotation.PostConstruct;
import java.util.concurrent.atomic.AtomicLong;

@RestController
public class HelloController {
    private AtomicLong atomicLong = new AtomicLong();
    private Counter counter;

    @Autowired
    private MetricRegistry metricRegistry;

    @PostConstruct
    public void init() {
        counter = metricRegistry.counter("counter");
    }

    @GetMapping("/hello")
    @Timed(name = "my-index")
    public String index() {
        counter.inc();

        return "Greetings from Spring Boot!. count=" + atomicLong.incrementAndGet();
    }

}

---

package com.example.demo;

import com.codahale.metrics.ConsoleReporter;
import com.codahale.metrics.MetricRegistry;
import com.codahale.metrics.jvm.FileDescriptorRatioGauge;
import com.codahale.metrics.jvm.GarbageCollectorMetricSet;
import com.codahale.metrics.jvm.MemoryUsageGaugeSet;
import com.codahale.metrics.jvm.ThreadStatesGaugeSet;
import com.codahale.metrics.servlets.AdminServlet;
import com.codahale.metrics.servlets.CpuProfileServlet;
import com.codahale.metrics.servlets.MetricsServlet;
import com.ryantenney.metrics.spring.config.annotation.EnableMetrics;
import com.ryantenney.metrics.spring.config.annotation.MetricsConfigurerAdapter;
import io.prometheus.client.dropwizard.DropwizardExports;
import org.springframework.boot.web.servlet.ServletRegistrationBean;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;

import java.util.concurrent.TimeUnit;

@Configuration
public class Config /*extends MetricsConfigurerAdapter*/ {
    //@Override
            //public void configureReporters(MetricRegistry metricRegistry) {
        //    // registerReporter allows the MetricsConfigurerAdapter to
        //    // shut down the reporter when the Spring context is closed
        //   // registerReporter(ConsoleReporter
        //   //         .forRegistry(metricRegistry)
        //   //         .build())
        //   //         .start(1, TimeUnit.MINUTES);


        //    new DropwizardExports(metricRegistry).register();
        //}

   @Bean
   public DropwizardExports dropwizardExports(MetricRegistry metricRegistry){
       DropwizardExports dropwizardExports = new DropwizardExports(metricRegistry);
       dropwizardExports.register();
       return dropwizardExports;
   }

    @Bean
    public MetricRegistry metricRegistry() {
        MetricRegistry metricRegistry = new MetricRegistry();
        metricRegistry.registerAll(new GarbageCollectorMetricSet());
        metricRegistry.registerAll(new MemoryUsageGaugeSet());
        metricRegistry.registerAll(new ThreadStatesGaugeSet());
        return metricRegistry;
    }

    @Bean
    public ConsoleReporter consoleReporter(MetricRegistry metricRegistry) {
        ConsoleReporter reporter = ConsoleReporter.forRegistry(metricRegistry).build();
        reporter.start(5, TimeUnit.SECONDS);
        reporter.report();
        return reporter;
    }

    @Bean
    public ServletRegistrationBean<MetricsServlet> registerMetricsServlet(MetricRegistry metricRegistry) {
        return new ServletRegistrationBean<>(new MetricsServlet(metricRegistry), "/metrics/*");
    }

    @Bean
    public ServletRegistrationBean<CpuProfileServlet> registerCpuServlet() {
        return new ServletRegistrationBean<>(new CpuProfileServlet(), "/cpu/*");
    }
}

build.gradle:

plugins {
    id 'org.springframework.boot' version '2.7.1'
    id 'io.spring.dependency-management' version '1.0.11.RELEASE'
    id 'java'
}

group = 'com.example'
version = '0.0.1-SNAPSHOT'
sourceCompatibility = '17'

repositories {
    mavenCentral()
}

dependencies {
    implementation 'org.springframework.boot:spring-boot-starter-web'
    implementation "org.springframework.boot:spring-boot-starter-actuator"
    // Minimum required for metrics.
    implementation ('com.ryantenney.metrics:metrics-spring:3.1.3') {
        exclude group: 'com.codahale.metrics'
        exclude group: 'org.springframework'
    }
    implementation 'io.dropwizard.metrics:metrics-core:4.2.9'
    implementation 'io.dropwizard.metrics:metrics-annotation:4.2.9'
    implementation 'io.dropwizard.metrics:metrics-servlets:4.2.9'

    implementation 'io.prometheus:simpleclient_dropwizard:0.15.0'
    implementation 'io.prometheus:simpleclient_servlet:0.15.0'
    implementation 'io.dropwizard:dropwizard-core:2.1.0'

    implementation 'com.ryantenney.metrics:metrics-spring:3.1.3'
    implementation 'io.prometheus:simpleclient_common:0.16.0'

    testImplementation 'org.springframework.boot:spring-boot-starter-test'
}

tasks.named('test') {
    useJUnitPlatform()
}

I access localhost:8080/metrics and receive following response:

{
  "version": "4.0.0",
  "gauges": {
    "G1-Old-Generation.count": {
      "value": 0
    },
    "G1-Old-Generation.time": {
      "value": 0
    },
    "G1-Young-Generation.count": {
      "value": 7
    },
    "G1-Young-Generation.time": {
      "value": 31
    },
    "blocked.count": {
      "value": 0
    },
    "count": {
      "value": 26
    },
    "daemon.count": {
      "value": 22
    },
    "deadlock.count": {
      "value": 0
    },
    "deadlocks": {
      "value": []
    },
    "heap.committed": {
      "value": 301989888
    },
    "heap.init": {
      "value": 532676608
    },
    "heap.max": {
      "value": 8518631424
    },
    "heap.usage": {
      "value": 0.008041180864688155
    },
    "heap.used": {
      "value": 68499856
    },
    "new.count": {
      "value": 0
    },
    "non-heap.committed": {
      "value": 51707904
    },
    "non-heap.init": {
      "value": 2555904
    },
    "non-heap.max": {
      "value": -1
    },
    "non-heap.usage": {
      "value": -5.0738536E7
    },
    "non-heap.used": {
      "value": 50738536
    },
    "peak.count": {
      "value": 32
    },
    "pools.CodeCache.committed": {
      "value": 10551296
    },
    "pools.CodeCache.init": {
      "value": 2555904
    },
    "pools.CodeCache.max": {
      "value": 50331648
    },
    "pools.CodeCache.usage": {
      "value": 0.2039642333984375
    },
    "pools.CodeCache.used": {
      "value": 10265856
    },
    "pools.Compressed-Class-Space.committed": {
      "value": 5177344
    },
    "pools.Compressed-Class-Space.init": {
      "value": 0
    },
    "pools.Compressed-Class-Space.max": {
      "value": 1073741824
    },
    "pools.Compressed-Class-Space.usage": {
      "value": 0.004625104367733002
    },
    "pools.Compressed-Class-Space.used": {
      "value": 4966168
    },
    "pools.G1-Eden-Space.committed": {
      "value": 188743680
    },
    "pools.G1-Eden-Space.init": {
      "value": 29360128
    },
    "pools.G1-Eden-Space.max": {
      "value": -1
    },
    "pools.G1-Eden-Space.usage": {
      "value": 0.26666666666666666
    },
    "pools.G1-Eden-Space.used": {
      "value": 50331648
    },
    "pools.G1-Eden-Space.used-after-gc": {
      "value": 0
    },
    "pools.G1-Old-Gen.committed": {
      "value": 109051904
    },
    "pools.G1-Old-Gen.init": {
      "value": 503316480
    },
    "pools.G1-Old-Gen.max": {
      "value": 8518631424
    },
    "pools.G1-Old-Gen.usage": {
      "value": 0.0017806278080379123
    },
    "pools.G1-Old-Gen.used": {
      "value": 15168512
    },
    "pools.G1-Old-Gen.used-after-gc": {
      "value": 15168512
    },
    "pools.G1-Survivor-Space.committed": {
      "value": 4194304
    },
    "pools.G1-Survivor-Space.init": {
      "value": 0
    },
    "pools.G1-Survivor-Space.max": {
      "value": -1
    },
    "pools.G1-Survivor-Space.usage": {
      "value": 0.7151832580566406
    },
    "pools.G1-Survivor-Space.used": {
      "value": 2999696
    },
    "pools.G1-Survivor-Space.used-after-gc": {
      "value": 2999696
    },
    "pools.Metaspace.committed": {
      "value": 35979264
    },
    "pools.Metaspace.init": {
      "value": 0
    },
    "pools.Metaspace.max": {
      "value": -1
    },
    "pools.Metaspace.usage": {
      "value": 0.9868604316086066
    },
    "pools.Metaspace.used": {
      "value": 35506512
    },
    "runnable.count": {
      "value": 10
    },
    "terminated.count": {
      "value": 0
    },
    "timed_waiting.count": {
      "value": 5
    },
    "total.committed": {
      "value": 353697792
    },
    "total.init": {
      "value": 535232512
    },
    "total.max": {
      "value": 8518631423
    },
    "total.used": {
      "value": 119238392
    },
    "total_started.count": {
      "value": 47
    },
    "waiting.count": {
      "value": 11
    }
  },
  "counters": {
    "counter": {
      "count": 9
    }
  },
  "histograms": {},
  "meters": {},
  "timers": {}
}

Obviously this output is not applicable for Prometheus (all dots should be replaced with "_" at least)

How can I make output in format ready for prometheus ?

PS

Based on documentation I've understand that class io.prometheus.client.dropwizardDropwizardExports is responsible for generating metric in format ready for Prometheus but I can't understand how.

Answer 1

I have implemented different ways to export metrics to Prometheus:

1.1) Custom implementation for Pushgateway

I wrote code that generates output into a StringBuilder and just follows the documentation https://prometheus.io/docs/instrumenting/exposition_formats/

Finally POST that string to the Pushgateway using a java HttpClient of your choice.

1.2) Custom page to be scraped by Prometheus

I wrote a dynamic page (servlet, jsp, ...) that generates output with plain/text content type and just follows the documentation https://prometheus.io/docs/instrumenting/exposition_formats/

Configure Prometheus to scrape that dynamic page.

2. use existing library

I made use of https://github.com/prometheus/client_java , which is the official client library for Java. Check the chapter about Exporting in the readme, it is quite good and covers both pushing the metrics as well as getting scraped.

Answer 2

I don't see significant difference but it is working example:

public class JavaDropwizard {
  // Create registry for Dropwizard metrics.
  static final MetricRegistry metrics = new MetricRegistry();
  // Create a Dropwizard counter.
  static final Counter counter = metrics.counter("my.example.counter.total");

  public static void main( String[] args ) throws Exception {
      // Increment the counter.
      counter.inc();

      // Hook the Dropwizard registry into the Prometheus registry
      // via the DropwizardExports collector.
      CollectorRegistry.defaultRegistry.register(new DropwizardExports(metrics));


      // Expose Prometheus metrics.
      Server server = new Server(1234);
      ServletContextHandler context = new ServletContextHandler();
      context.setContextPath("/");
      server.setHandler(context);
      context.addServlet(new ServletHolder(new MetricsServlet()), "/metrics");
      // Add metrics about CPU, JVM memory etc.
      DefaultExports.initialize();
      // Start the webserver.
      server.start();
      server.join();
  }
}

pom.xml

<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
  xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
  <modelVersion>4.0.0</modelVersion>
  <groupId>io.robustperception.java_examples</groupId>
  <artifactId>java_dropwizard</artifactId>
  <packaging>jar</packaging>
  <version>1.0-SNAPSHOT</version>
  <properties>
    <maven.compiler.source>17</maven.compiler.source>
    <maven.compiler.target>17</maven.compiler.target>
  </properties>
  <name>java_dropwizard</name>
  <url>http://maven.apache.org</url>
  <dependencies>
    <dependency>
      <groupId>io.prometheus</groupId>
      <artifactId>simpleclient</artifactId>
      <version>0.15.0</version>
    </dependency>
    <dependency>
      <groupId>io.prometheus</groupId>
      <artifactId>simpleclient_hotspot</artifactId>
      <version>0.15.0</version>
    </dependency>
    <dependency>
      <groupId>io.prometheus</groupId>
      <artifactId>simpleclient_servlet</artifactId>
      <version>0.15.0</version>
    </dependency>
    <dependency>
      <groupId>io.prometheus</groupId>
      <artifactId>simpleclient_dropwizard</artifactId>
      <version>0.15.0</version>
    </dependency>
    <dependency>
        <groupId>io.dropwizard.metrics</groupId>
        <artifactId>metrics-core</artifactId>
        <version>4.2.9</version>
    </dependency>
    <dependency>
      <groupId>org.eclipse.jetty</groupId>
      <artifactId>jetty-servlet</artifactId>
      <version>8.1.7.v20120910</version>
    </dependency>
  </dependencies>

  <build>
    <plugins>
      <plugin>
        <groupId>org.apache.maven.plugins</groupId>
        <artifactId>maven-compiler-plugin</artifactId>
        <version>3.1</version>
      </plugin>
      <!-- Build a full jar with dependencies --> 
      <plugin>
        <artifactId>maven-assembly-plugin</artifactId>
        <configuration>
          <archive>
        <manifest>
          <mainClass>io.robustperception.java_examples.JavaDropwizard</mainClass>
        </manifest>
      </archive>
      <descriptorRefs>
        <descriptorRef>jar-with-dependencies</descriptorRef>
      </descriptorRefs>
    </configuration>
    <executions>
      <execution>
        <id>make-assembly</id>
        <phase>package</phase>
        <goals>
          <goal>single</goal>
        </goals>
      </execution>
    </executions>
  </plugin>
</plugins>

Answer 3

Finally I've found a root cause:

com.codahale.metrics.servlets.MetricsServlet

should be replaced with

io.prometheus.client.exporter.MetricsServlet