how can I reduce cpu usage in a golang tcp server?

Question

I try to implement a golang tcp server, and I found the concurrency is satisfied for me, but the CPU usage is too high(concurrency is 15W+/s, but the CPU usage is about 800% in a 24 cores linux machine). At the same time, a C++ tcp server is only about 200% usage with a similar concurrency(with libevent).

The following code is the demo of golang:

func main() {
    listen, err := net.Listen("tcp", "0.0.0.0:17379")
    if err != nil {
        fmt.Errorf(err.Error())
    }
    go acceptClient(listen)
    var channel2 = make(chan bool)
    <-channel2
}

func acceptClient(listen net.Listener) {
    for {
        sock, err := listen.Accept()
        if err != nil {
            fmt.Errorf(err.Error())
        }
        tcp := sock.(*net.TCPConn)
        tcp.SetNoDelay(true)
        var channel = make(chan bool, 10)
        go read(channel, sock.(*net.TCPConn))
        go write(channel, sock.(*net.TCPConn))
    }
}

func read(channel chan bool, sock *net.TCPConn) {
    count := 0
    for {
        var buf = make([]byte, 1024)
        n, err := sock.Read(buf)
        if err != nil {
            close(channel)
            sock.CloseRead()
            return
        }
        count += n
        x := count / 58
        count = count % 58
        for i := 0; i < x; i++ {
            channel <- true
        }
   }
}

func write(channel chan bool, sock *net.TCPConn) {
    buf := []byte("+OK\r\n")
    defer func() {
        sock.CloseWrite()
        recover()
    }()
    for {
        _, ok := <-channel
        if !ok {
            return
        }
        _, writeError := sock.Write(buf)
        if writeError != nil {
            return
        }
    }
}

And I test this tcp server by the redis-benchmark with multi-clients:

redis-benchmark -h 10.100.45.2  -p 17379 -n 1000 -q script load "redis.call('set','aaa','aaa')"

I also analyzed my golang code by the pprof, it is said CPU cost a lot of time on syscall: enter image description here

Answer 1

I don't think parallelise the read and write with channel will provide you better performance in this case. You should try to do less memory allocation and less syscall (The write function may do a lot of syscalls)

Can you try this version?

package main

import (
    "bytes"
    "fmt"
    "net"
)

func main() {
    listen, err := net.Listen("tcp", "0.0.0.0:17379")
    if err != nil {
        fmt.Errorf(err.Error())
    }
    acceptClient(listen)
}

func acceptClient(listen net.Listener) {
    for {
        sock, err := listen.Accept()
        if err != nil {
            fmt.Errorf(err.Error())
        }
        tcp := sock.(*net.TCPConn)
        tcp.SetNoDelay(true)
        go handleConn(tcp) // less go routine creation but no concurrent read/write on the same conn
    }
}

var respPattern = []byte("+OK\r\n")

// just one goroutine per conn
func handleConn(sock *net.TCPConn) {
    count := 0
    buf := make([]byte, 4098) // Do not create a new buffer each time & increase the buff size
    defer sock.Close()

    for {
        n, err := sock.Read(buf)
        if err != nil {
            return
        }
        count += n
        x := count / 58
        count = count % 58
        resp := bytes.Repeat(respPattern, x) // can be optimize
        _, writeError := sock.Write(resp) // do less syscall
        if writeError != nil {
            return
        }
    }
}