How do I close a channel and wait for multiple http responses?

Question

I have the following and am trying to make some concurrent http calls so that I can speed up the entire program rather than doing each call 1 by one:

package main

import (
    "fmt"
    "net/http"
    "time"
)

type U struct {
    u    string
    name string
    resp *http.Response
}

func main() {
    urls := []*U{
        &U{"example", "http://www.example.com", nil},
        &U{"yahoo", "http://www.yahoo.com", nil},
        &U{"google", "http://www.google.com", nil},
    }

    ch := make(chan *U)

    // read from the channel
    go func() {
        for c := range ch {
            for i, u := range urls {
                if c.name == u.name {
                    urls[i] = c
                }
            }
        }
    }()

    // fetch the stuff
    for _, u := range urls {
        go func(u *U) {
            var err error
            u, err = getResponse(u)
            if err != nil {
                fmt.Println(err)
            }
            ch <- u
        }(u)
    }

    for i, u := range urls {
        fmt.Println(i, u.resp) // all nil
    }
}

func getResponse(u *U) (*U, error) {
    c := &http.Client{
        Timeout: 10 * time.Second,
    }

    var err error
    u.resp, err = c.Get(u.u)

    return u, err
}

https://play.golang.org/p/Zko8xkEqDMB

I am ovbiously not doing something right as it prints

0 <nil>
1 <nil>
2 <nil>

whereas it should print the response as well.

How do I make sure I wait until everything is done so that I can move forward? ok. thanks.

Answer 1

You're not waiting for the responses to return. This is equivalent to:

urls := []*U{
    &U{"example", "http://www.example.com", nil},
    &U{"yahoo", "http://www.yahoo.com", nil},
    &U{"google", "http://www.google.com", nil},
}

for i, u := range urls {
    fmt.Println(i, u.resp) // all nil
}

Instead you could use a sync.WaitGroup to make sure all the work is done before you display responses:

var wg sync.WaitGroup

for _, u := range urls {
    wg.Add(1)  // Add job to the waitgroup
    go func(u *U) {
        var err error
        u, err = getResponse(u)
        if err != nil {
            fmt.Println(err)
        }
        ch <- u
        wg.Done()  // Note when the job is done
    }(u)
}

wg.Wait()  // wait until all the Add'd jobs are Done'd
for i, u := range urls {
    fmt.Println(i, u.resp) // all nil
}

Or you could handle printing responses in the same chain as getResponse :

for _, u := range urls {
    go func(u *U) {
        var err error
        u, err = getResponse(u)
        if err != nil {
            fmt.Println(err)
        }
        printResponse(u)  // implement printResponse however
        ch <- u
    }(u)
}