Golang协程和mysql

Question

我试图通过 goroutines 查询数据库，但我经常遇到查询数量的限制，我无法以任何方式解决这个问题:(

但每次我有“阅读：连接重置”或“连接太多”

我究竟做错了什么？ 请帮帮我。 提前致谢。 这是我的代码。

type Page struct {
    Stat int
}

func main() {
    cfg := mysql.Config{
        // some config
    }
    // 5000 ids
    groups := []int{}

    // trying set buffer limit
    pages := make(chan Page, 8)
    for _, id := range groups {
        go getData(id, cfg, pages)
    }
    for _, id := range groups {
        page := <-pages
        fmt.Println(id, page.Stat)
    }
}

func getData(i int, cfg mysql.Config, channel chan Page) {
    db, err := sql.Open("mysql", cfg.FormatDSN())
    db.SetMaxOpenConns(8)
    db.SetMaxIdleConns(8)
    checkError(err)
    rows, err := db.Query(`select g.id from goods as g where g.groupid = ?`, i)
    checkError(err)

    defer rows.Close()
    defer db.Close()

    count := 0;
    for rows.Next() {
        err = rows.Scan(&id1)
        checkError(err)
        count++
    }
    channel <- Page{Stat: count}
}

Answer 1

连接创建应该在 getData 之外完成。 此代码可以并行创建太多连接（~ 5000）。

Answer 2

根据评论，数据库实例应该在 goroutines 之外处理。 但是由于你在连接池中对打开连接的设置，单独这样做仍然会导致错误。 例如，您将最大打开连接数设置为 8，然后生成所有 goroutine。 某些 goroutine 可能会在等待来自连接池的可用连接时超时。

您可以改进这一点并优化连接池中连接的使用和 goroutine 的数量以获得最佳的无错误性能。 一个时刻可以激活的 goroutine 的最大数量应该等于可用的打开连接的数量，只是为了安全起见不要导致任何超时错误。

func main() {

    cfg := mysql.Config{
        // some config
    }

    numberOfConns := 20
    db, err := sql.Open("mysql", cfg.FormatDSN())
    checkError(err)
    
    defer db.Close()
    db.SetMaxOpenConns(numberOfConns)
    db.SetMaxIdleConns(numberOfConns)

    // 5000 ids
    groups := []int{}

    // trying set buffer limit
    pages := make(chan Page, numberOfConns)
    limit := make(chan bool, numberOfConns)
    for _, id := range groups {
        limit <- true //limits the number of goroutines that can be spawned based on numberOfConns 
        go getData(id, db, pages, limit)
    }
    for _, id := range groups {
        page := <-pages
        fmt.Println(id, page.Stat)
    }
}

func getData(i int, db *sql.DB, channel chan Page, limit chan bool) {
    rows, err := db.Query(`select g.id from goods as g where g.groupid = ?`, i)
    checkError(err)

    defer rows.Close()

    count := 0;
    for rows.Next() {
        err = rows.Scan(&id1)
        checkError(err)
        count++
    }
    channel <- Page{Stat: count}
    <-limit // release the resource so next goroutine can be started
}

Answer 3

这是您的代码的正确版本。

type Page struct {
    Stat int
    Id   int
}

func main() {
    cfg := mysql.Config{
        // some config
    }
    // 5000 ids
    groups := []int{1, 2, 3}

    // this buffered channel will block at the concurrency limit
    semaphoreChan := make(chan struct{}, 5)

    // for collecting result
    pages := make(chan *Page)

    defer func() {
        close(semaphoreChan)
        close(pages)
    }()

    go func() {
        for _, id := range groups {
            // this sends an empty struct into the semaphoreChan which
            // is basically saying add one to the limit, but when the
            // limit has been reached block until there is room
            semaphoreChan <- struct{}{}

            go func(i int) {

                getData(i, cfg, pages)
                // once we're done it's we read from the semaphoreChan which
                // has the effect of removing one from the limit and allowing
                // another goroutine to start
                <-semaphoreChan

            }(id)
        }
    }()

    var results []Page

    for {
        result := <-pages
        log.Println(result.Stat, result.Id)
        results = append(results, *result)
        if len(results) == len(groups) {
            break
        }
    }

}

func getData(i int, cfg mysql.Config, channel chan *Page) {
    db, err := sql.Open("mysql", cfg.FormatDSN())
    db.SetMaxOpenConns(8)
    db.SetMaxIdleConns(8)
    checkError(err)
    rows, err := db.Query(`select g.id from goods as g where g.groupid = ?`, i)
    checkError(err)

    defer rows.Close()
    defer db.Close()

    count := 0
    for rows.Next() {
        err = rows.Scan(&id1)
        checkError(err)
        count++
    }
    channel <- &Page{Stat: count, Id: i}
}