Can't access my app deployed on an AKS cluster, from the azure public ip

Question

I'm very new to Cloud providers and I'm trying to setup a cluster on Azure. I setup the infra with terraform, and I create a ResourceGroup, a static Azure PublicIP (sku basic), a Azure Load Balancer(sku basic, connected to the public ip, front end port: 80, backend port 3000) and a AKS cluster. With Skaffold I then deploy on the AKS cluster a ClusterIP ( port: 3000, targetPort: 3000 that selects the server pods), the server Deployment( which listens to port 3000) plus the secrets.

The deployment goes well and logging the server pods shows the app is running correctly and listening on por 3000, but when I try to access the server either with the address ( 20.218.249.246:80/api where 80 is the load balancer front end port and the /api is the base for the router) or the dns (fixit.germanywestcentral.cloudapp.azure.com:80/api) from the Azure console the connection fails after timeout. I deployed a Kube.netes Load balancer to test the cluster and from its external ip I can indeed access the server. looking in the troubleshot guides I see that the check on the cluster's load balancer passes as it uses a Standard load balancer, but I did create a Basic one in terraform.

load balance check public ip load balancer cluster resource group

It would seem by the check that my cluster is not using the load balancer I created as the one it is using is Standard and the one I created is Basic.

Am I missing out something in setting up the cluster or the ip on Azure? Many thanks for the help, here are the files

resource group

resource "azurerm_resource_group" "resource_group" {
  name     = var.resource_group_name
  location = var.location
    tags = {
    Environment = "Production"
    Team = "DevOps"
  }
}

public ip

resource "azurerm_public_ip" "public-ip" {
  name                = "fixit-public-ip"
  location            = var.location
  resource_group_name = var.resource_group_name
  allocation_method   = "Static"
  domain_name_label = "fixit"
  # sku = "Standard"
  # fixit.germanywestcentral.cloudapp.azure.com

}

load balancer

resource "azurerm_lb" "load-balancer" {
  name                = "fixit-load-balancer"
  location            = var.location
  resource_group_name = var.resource_group_name
  # sku = "Standard"

  frontend_ip_configuration {
    name                 = "PublicIPAddress"
    public_ip_address_id = azurerm_public_ip.public-ip.id
  }
  
  
}
resource "azurerm_lb_backend_address_pool" "address-pool" {
  name                = "fixit-backend-pool"
  loadbalancer_id     = azurerm_lb.load-balancer.id
  
}

resource "azurerm_lb_rule" "load-balancer-rule" {
  name = "fixit-load-balancer-rule"
  loadbalancer_id = azurerm_lb.load-balancer.id
  frontend_ip_configuration_name = "PublicIPAddress"
  protocol                       = "Tcp"
  frontend_port                  = 80
  # backend_port                   = 27017
  backend_port                   = 3000
  # disable_outbound_snat = true

}

cluster

resource "azurerm_kubernetes_cluster" "server_cluster" {
  name                = "server_cluster"
  location            = azurerm_resource_group.resource_group.location
  resource_group_name = azurerm_resource_group.resource_group.name
  dns_prefix          = "fixit"
  kubernetes_version = var.kubernetes_version
  # sku_tier = "Paid"

  default_node_pool {
    name       = "default"
    node_count = 1
    min_count = 1
    max_count = 3
    # vm_size    = "standard_b2s_v5"
    vm_size    = "standard_e2bs_v5"
    type = "VirtualMachineScaleSets"
    enable_auto_scaling = true
    enable_host_encryption = false
    # os_disk_size_gb = 30
    enable_node_public_ip = true
    
  }

  identity {
    type = "SystemAssigned"
  }

  tags = {
    Environment = "Production"
  }

  linux_profile {
    admin_username = "azureuser"
    ssh_key {
        key_data = var.ssh_key
    }
  }
  network_profile {
      network_plugin = "kubenet"
      # load_balancer_sku = "standard"
    
  }
  
}

cluster ip

apiVersion: v1
kind: Service
metadata:
  name: server-clusterip-service
spec:
  type: ClusterIP cloud provider's load balancer
  selector:
    app: fixit-server-pod
  ports:
    - name: server-clusterip-service
      protocol: TCP
      port: 3000 # service port
      targetPort: 3000 # por on which the app is listening to

server

apiVersion: apps/v1
kind: Deployment
metadata:
  name: fixit-server
spec:
  replicas: 1
  revisionHistoryLimit: 10
  selector:
    matchLabels:
      app: fixit-server-pod
  template:
    metadata:
      labels:
        app: fixit-server-pod
    spec:
      imagePullSecrets:
        - name: docker-secret
      containers:
        - name: fixit-server-container
          image: vinnytwice/fixit-server:dev
          imagePullPolicy: 'Always'
          env:
            # - name: SERVER_DEV_IMAGE_TAG
            #   value: 'dev'

            # server
            - name: APP_LISTENING_PORT
              value: '3000'
            - name: API_KEY
              valueFrom:
                secretKeyRef:
                  name: server-secret
                  key: api-key
            # stripe
            - name: STRIPE_KEY
              valueFrom:
                secretKeyRef:
                  name: stripe-secret
                  key: stripe-key
            # mongo db connection string
            - name: MONGO_USERNAME_K8S
              valueFrom:
                secretKeyRef:
                  key: mongo-username-k8s
                  name: server-secret

            - name: MONGO_HOSTNAME_K8S
              valueFrom:
                secretKeyRef:
                  key: mongo-hostname-k8s
                  name: server-secret

            - name: MONGO_PORT_K8S
              valueFrom:
                secretKeyRef:
                  name: server-secret
                  key: mongo-port-k8s

            # neo4j connection string
            - name: MONGO_DB_K8S
              valueFrom:
                secretKeyRef:
                  key: mongo-db-k8s
                  name: server-secret

            - name: NEO4J_AURA_URI
               valueFrom:
                 secretKeyRef:
                   key: neo4j-aura-uri
                   name: neo4j-secret
            - name: NEO4J_AURA_USERNAME
               valueFrom:
                 secretKeyRef:
                   key: neo4j-aura-username
                   name: neo4j-secret
            - name: NEO4J_AURA_PASSWORD
               valueFrom:
                 secretKeyRef:
                   key: neo4j-aura-password
                   name: neo4j-secret
          resources:
            limits:
              memory: '2Gi'
              cpu: '500m'
              # cpu: '1.0'

Answer 1

I finally found the problem. It was my setup of course. I wasn't installing any ingress controller so of course the ingress service wasn't working at all, the proper implementation is to install an ingress controller https://learn.microsoft.com/en-us/azure/aks/ingress-basic?tabs=azure-cli , which hooks up the the internal cluster load balancer, so there is no need to create one. Now, the ingress controller will accept an ip address for the load balancer, but you have to create the PublicIP in the node resource group because is going to look for it there and not in the resource group , got it after checking the difference between the two here https://learn.microsoft.com/en-us/azure/aks/faq#why-are-two-resource-groups-created-with-aks .

So the working configuration is now:

main

terraform {
  required_version = ">=1.1.0"
  required_providers {
    azurerm = {
      source = "hashicorp/azurerm"
       version = "~> 3.0.2"
    }
  }
}

provider "azurerm" {
  features {
    resource_group {
      prevent_deletion_if_contains_resources = false
    }
  }
  subscription_id   = var.azure_subscription_id
  tenant_id         = var.azure_subscription_tenant_id
  client_id         = var.service_principal_appid
  client_secret     = var.service_principal_password
}


provider "kubernetes" {
  host = "${module.cluster.host}"
  client_certificate = "${base64decode(module.cluster.client_certificate)}"
  client_key = "${base64decode(module.cluster.client_key)}"
  cluster_ca_certificate = "${base64decode(module.cluster.cluster_ca_certificate)}"
}

provider "helm" {
  kubernetes {
    host                   = "${module.cluster.host}"
    client_certificate     = "${base64decode(module.cluster.client_certificate)}"
    client_key             = "${base64decode(module.cluster.client_key)}"
    cluster_ca_certificate = "${base64decode(module.cluster.cluster_ca_certificate)}"
  }
}



module "cluster" {
  source = "./modules/cluster"
  location = var.location
  vm_size = var.vm_size
  resource_group_name = var.resource_group_name
  node_resource_group_name = var.node_resource_group_name
  kubernetes_version = var.kubernetes_version
  ssh_key = var.ssh_key
  sp_client_id = var.service_principal_appid
  sp_client_secret = var.service_principal_password
}




module "ingress-controller" {
  source = "./modules/ingress-controller"
  public_ip_address = module.cluster.public_ip_address
  depends_on = [
    module.cluster.public_ip_address
  ]
}

cluster

resource "azurerm_resource_group" "resource_group" {
  name     = var.resource_group_name
  location = var.location
    tags = {
    Environment = "test"
    Team = "DevOps"
  }
}
resource "azurerm_kubernetes_cluster" "server_cluster" {
  name                = "server_cluster"
  ### choose the resource goup to use for the cluster
  location            = azurerm_resource_group.resource_group.location
  resource_group_name = azurerm_resource_group.resource_group.name
  ### decide the name of the cluster "node" resource group, if unset will be named automatically 
  node_resource_group = var.node_resource_group_name
  dns_prefix          = "fixit"
  kubernetes_version = var.kubernetes_version
  # sku_tier = "Paid"

  default_node_pool {
    name       = "default"
    node_count = 1
    min_count = 1
    max_count = 3
    vm_size = var.vm_size

    type = "VirtualMachineScaleSets"
    enable_auto_scaling = true
    enable_host_encryption = false
    # os_disk_size_gb = 30
  }

  service_principal {
    client_id = var.sp_client_id
    client_secret = var.sp_client_secret
  }

  tags = {
    Environment = "Production"
  }

  linux_profile {
    admin_username = "azureuser"
    ssh_key {
        key_data = var.ssh_key
    }
  }
  network_profile {
      network_plugin = "kubenet"
      load_balancer_sku = "basic" 
    
  }
  http_application_routing_enabled = false
  depends_on = [
    azurerm_resource_group.resource_group
  ]
}

resource "azurerm_public_ip" "public-ip" {
  name                = "fixit-public-ip"
  location            = var.location
  # resource_group_name = var.resource_group_name
  resource_group_name = var.node_resource_group_name
  allocation_method   = "Static"
  domain_name_label = "fixit"
  # sku = "Standard"

depends_on = [
  azurerm_kubernetes_cluster.server_cluster
]
}

ingress controller

resource "helm_release" "nginx" {
  name      = "ingress-nginx"
  repository = "ingress-nginx"
  chart     = "ingress-nginx/ingress-nginx"
  namespace = "default"

  set {
    name  = "controller.service.externalTrafficPolicy"
    value = "Local"
  }

  set {
    name = "controller.service.annotations.service.beta.kubernetes.io/azure-load-balancer-internal"
    value = "true"
  }

  set {
    name  = "controller.service.loadBalancerIP"
    value = var.public_ip_address
  }

  set {
    name  = "controller.service.annotations.service.beta.kubernetes.io/azure-load-balancer-health-probe-request-path"
    value = "/healthz"
  }
} 

ingress service

apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: ingress-service
  # namespace: default
  annotations:
    nginx.ingress.kubernetes.io/ssl-redirect: "false"
    nginx.ingress.kubernetes.io/use-regex: "true"
    nginx.ingress.kubernetes.io/rewrite-target: /$2$3$4
spec:
  ingressClassName: nginx
  rules:
    # - host: fixit.westeurope.cloudapp.azure.com #dns from Azure PublicIP


### Node.js server
  - http:
      paths:
      - path: /(/|$)(.*)
        pathType: Prefix
        backend:
          service:
            name: server-clusterip-service
            port:
              number: 80 

  - http:
      paths:
      - path: /server(/|$)(.*)
        pathType: Prefix
        backend:
          service:
            name: server-clusterip-service
            port:
              number: 80
...

other services omitted

Hope this can help getting the setup right. Cheers.