How can I get a Kubernetes clientset in GO using a JSON service account key?

Question

I need to create a Kubernetes clientset using a token extracted from JSON service account key file.

I explicitly provide this token inside the config, however it still looks for Google Application-Default credentials, and crashes because it cannot find them.

Below is my code:

package main

import (
    "context"
    "encoding/base64"
    "fmt"
    "io/ioutil"

    "golang.org/x/oauth2"
    "golang.org/x/oauth2/google"
    gke "google.golang.org/api/container/v1"
    "google.golang.org/api/option"
    "k8s.io/client-go/kubernetes"
    _ "k8s.io/client-go/plugin/pkg/client/auth/gcp"
    "k8s.io/client-go/tools/clientcmd"
    "k8s.io/client-go/tools/clientcmd/api"
)

const (
    projectID   = "my_project_id"
    clusterName = "my_cluster_name"
    scope       = "https://www.googleapis.com/auth/cloud-platform"
)

func main() {
    ctx := context.Background()

    // Read JSON key and extract the token
    data, err := ioutil.ReadFile("sa_key.json")
    if err != nil {
        panic(err)
    }
    creds, err := google.CredentialsFromJSON(ctx, data, scope)
    if err != nil {
        panic(err)
    }
    token, err := creds.TokenSource.Token()
    if err != nil {
        panic(err)
    }
    fmt.Println("token", token.AccessToken)

    // Create GKE client
    tokenSource := oauth2.StaticTokenSource(token)
    gkeClient, err := gke.NewService(ctx, option.WithTokenSource(tokenSource))
    if err != nil {
        panic(err)
    }

    // Create a dynamic kube config
    inMemKubeConfig, err := createInMemKubeConfig(ctx, gkeClient, token, projectID)
    if err != nil {
        panic(err)
    }

    // Use it to create a rest.Config
    config, err := clientcmd.NewNonInteractiveClientConfig(*inMemKubeConfig, clusterName, &clientcmd.ConfigOverrides{CurrentContext: clusterName}, nil).ClientConfig()
    if err != nil {
        panic(err)
    }

    // Create the clientset
    clientset, err := kubernetes.NewForConfig(config)
    if err != nil {
        panic(err) // this where the code crashes because it can't find the Google ADCs
    }

    fmt.Printf("clientset %+v\n", clientset)
}

func createInMemKubeConfig(ctx context.Context, client *gke.Service, token *oauth2.Token, projectID string) (*api.Config, error) {
    k8sConf := api.Config{
        APIVersion: "v1",
        Kind:       "Config",
        Clusters:   map[string]*api.Cluster{},
        AuthInfos:  map[string]*api.AuthInfo{},
        Contexts:   map[string]*api.Context{},
    }

    // List all clusters in project with id projectID across all zones ("-")
    resp, err := client.Projects.Zones.Clusters.List(projectID, "-").Context(ctx).Do()
    if err != nil {
        return nil, err
    }

    for _, f := range resp.Clusters {
        name := fmt.Sprintf("gke_%s_%s_%s", projectID, f.Zone, f.Name) // My custom naming convention
        cert, err := base64.StdEncoding.DecodeString(f.MasterAuth.ClusterCaCertificate)
        if err != nil {
            return nil, err
        }

        k8sConf.Clusters[name] = &api.Cluster{
            CertificateAuthorityData: cert,
            Server:                   "https://" + f.Endpoint,
        }

        k8sConf.Contexts[name] = &api.Context{
            Cluster:  name,
            AuthInfo: name,
        }

        k8sConf.AuthInfos[name] = &api.AuthInfo{
            Token: token.AccessToken,
            AuthProvider: &api.AuthProviderConfig{
                Name: "gcp",
                Config: map[string]string{
                    "scopes": scope,
                },
            },
        }
    }
    return &k8sConf, nil
}

and here is the error message:

panic: cannot construct google default token source: google: could not find default credentials. See https://developers.google.com/accounts/docs/application-default-credentials for more information.

Answer 1

You're using clusterName twice where you've specified that the Context name is gke_%s_%s_%s .

You're Contexts are named gke_%s_%s_%s :

name := fmt.Sprintf("gke_%s_%s_%s", projectID, f.Zone, f.Name)

k8sConf.Contexts[name] = &api.Context{
  Cluster:  name,
  AuthInfo: name,
}

But, when you clientcmd.NewNonInteractiveConfig , you're using clusterName as the context parameter and then again as the CurrentContext override.

clientcmd.NewNonInteractiveClientConfig(
  *inMemKubeConfig,
  clusterName,                   // Incorrect should be `gke_%s_%s_%s`
  &clientcmd.ConfigOverrides{
    CurrentContext: clusterName, // Redudant and incorrect
  },
  nil,
).ClientConfig()

You want to use the equivalent gke_%s_%s_%s name. Or, use the cluster name when you define the context name 's.

config, err := clientcmd.NewNonInteractiveClientConfig(
  *inMemKubeConfig,
  "gke_%s_%s_%s", // You need to replace this with the Context name
  &clientcmd.ConfigOverrides{},
  clientcmd.DefaultClientConfig.ConfigAccess(),
).ClientConfig()

Answer 2

Here's what worked for me.

It is based on thisgist and it's exactly what I was looking for. It uses an oauth2.TokenSource object which can be fed with a variety of token types so it's quite flexible.

It took me a long time to find this solution so I hope this helps somebody!

package main

import (
    "context"
    "encoding/base64"
    "fmt"
    "io/ioutil"
    "log"
    "net/http"

    gke "google.golang.org/api/container/v1"
    "google.golang.org/api/option"

    "golang.org/x/oauth2"
    "golang.org/x/oauth2/google"
    metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    "k8s.io/client-go/kubernetes"
    "k8s.io/client-go/rest"
    clientcmdapi "k8s.io/client-go/tools/clientcmd/api"
)

const (
    googleAuthPlugin = "gcp"
    projectID        = "my_project"
    clusterName      = "my_cluster"
    zone             = "my_cluster_zone"
    scope            = "https://www.googleapis.com/auth/cloud-platform"
)

type googleAuthProvider struct {
    tokenSource oauth2.TokenSource
}

// These funcitons are needed even if we don't utilize them
// So that googleAuthProvider is an rest.AuthProvider interface
func (g *googleAuthProvider) WrapTransport(rt http.RoundTripper) http.RoundTripper {
    return &oauth2.Transport{
        Base:   rt,
        Source: g.tokenSource,
    }
}
func (g *googleAuthProvider) Login() error { return nil }


func main() {
    ctx := context.Background()

    // Extract a token from the JSON SA key
    data, err := ioutil.ReadFile("sa_key.json")
    if err != nil {
        panic(err)
    }
    creds, err := google.CredentialsFromJSON(ctx, data, scope)
    if err != nil {
        panic(err)
    }
    token, err := creds.TokenSource.Token()
    if err != nil {
        panic(err)
    }
    tokenSource := oauth2.StaticTokenSource(token)

    // Authenticate with the token
    // If it's nil use Google ADC
    if err := rest.RegisterAuthProviderPlugin(googleAuthPlugin,
        func(clusterAddress string, config map[string]string, persister rest.AuthProviderConfigPersister) (rest.AuthProvider, error) {
            var err error
            if tokenSource == nil {
                tokenSource, err = google.DefaultTokenSource(ctx, scope)
                if err != nil {
                    return nil, fmt.Errorf("failed to create google token source: %+v", err)
                }
            }
            return &googleAuthProvider{tokenSource: tokenSource}, nil
        }); err != nil {
        log.Fatalf("Failed to register %s auth plugin: %v", googleAuthPlugin, err)
    }

    gkeClient, err := gke.NewService(ctx, option.WithTokenSource(tokenSource))
    if err != nil {
        panic(err)
    }

    clientset, err := getClientSet(ctx, gkeClient, projectID, org, env)
    if err != nil {
        panic(err)
    }

    // Demo to make sure it works
    pods, err := clientset.CoreV1().Pods("").List(ctx, metav1.ListOptions{})
    if err != nil {
        panic(err)
    }
    log.Printf("There are %d pods in the cluster", len(pods.Items))
    for _, pod := range pods.Items {
        fmt.Println(pod.Name)
    }
}

func getClientSet(ctx context.Context, client *gke.Service, projectID, name string) (*kubernetes.Clientset, error) {
    // Get cluster info
    cluster, err := client.Projects.Zones.Clusters.Get(projectID, zone, name).Context(ctx).Do()
    if err != nil {
        panic(err)
    }

    // Decode cluster CA certificate
    cert, err := base64.StdEncoding.DecodeString(cluster.MasterAuth.ClusterCaCertificate)
    if err != nil {
        return nil, err
    }

    // Build a config using the cluster info
    config := &rest.Config{
        TLSClientConfig: rest.TLSClientConfig{
            CAData: cert,
        },
        Host:         "https://" + cluster.Endpoint,
        AuthProvider: &clientcmdapi.AuthProviderConfig{Name: googleAuthPlugin},
    }

    return kubernetes.NewForConfig(config)
}