Kotlin Ktor client 403 forbidden cloudflare

Question

So im trying to request a side with proxies that is protected with cloudflare. The problem is i get 403 forbidden cloduflare error but only when im using proxies without it works. But the proxies are not the problem i tried them with python(requests module) and in my browser there i dont get blocked. My code

suspend fun scrape() {
    val client = HttpClient {
        followRedirects = true

        install(ContentNegotiation) {
            json(Json {
                ignoreUnknownKeys = true
            })
        }
        engine {
            proxy =
                ProxyBuilder.http("http://ProxyIP:proxyPort")

        }
        defaultRequest {
            val credentials = Base64.getEncoder().encodeToString("ProxyUser:ProxyPassword".toByteArray())
            header(HttpHeaders.ProxyAuthorization, "Basic $credentials")

        }



    }
    val response = client.get("http://example.com")
    val body = response.bodyAsText()
    println(body)
    println(response.status.hashCode())

Fixxed it

suspend fun scrape() {
    val client = HttpClient(Apache) {

        install(ContentNegotiation) {
            json(Json {
                ignoreUnknownKeys = true
            })
        }
        engine {
            followRedirects = false
            customizeClient  {
                setProxy(HttpHost("hostname", port))

                val credentialsProvider = BasicCredentialsProvider()
                credentialsProvider .setCredentials(
                    AuthScope("hostname", port),
                    UsernamePasswordCredentials("username", "password")
                )
                setDefaultCredentialsProvider(credentialsProvider )


            }
        }

    }

    val response =
        client.get("http://example.com") {
        }
    val body = response.bodyAsText()
    println(body)
    println(response.status.hashCode())
}

Answer 1

There is a problem with making a request through a proxy server using the CIO engine. I've created an issue to address this problem. As a workaround, please use the OkHttp engine instead of CIO .

Here is how you can use a proxy with the Basic authentication:

import io.ktor.client.*
import io.ktor.client.engine.okhttp.*
import io.ktor.client.request.*
import kotlinx.coroutines.runBlocking
import okhttp3.Authenticator
import okhttp3.Credentials
import okhttp3.OkHttpClient
import java.net.Proxy


fun main(): Unit = runBlocking {
    val proxyAuthenticator = Authenticator { _, response ->
        response.request.newBuilder()
            .header("Proxy-Authorization", Credentials.basic("<username>", "<password>"))
            .build()
    }

    val client = HttpClient(OkHttp) {
        engine {
            preconfigured = OkHttpClient.Builder()
                .proxy(Proxy(Proxy.Type.HTTP, java.net.InetSocketAddress("127.0.0.1", 3128)))
                .proxyAuthenticator(proxyAuthenticator)
                .build()
        }
    }

    val response = client.get("http://eu.kith.com/products.json")
    println(response.status)
}