Golang使用goroutines并行下载多个文件

Question

Is it possible to download and save files in parallel using goroutines?

Below is my code which downloads files from my dropbox:

package main

import (
    "encoding/json"
    "fmt"
    "io"
    "io/ioutil"
    "net/http"
    "net/url"
    "os"
    "path/filepath"
)

const app_key string = "<app_key>"
const app_secret string = "<app_secret>"

var code string

type TokenResponse struct {
    AccessToken string `json:"access_token"`
}

type File struct {
    Path string
}

type FileListResponse struct {
    FileList []File `json:"contents"`
}

func download_file(file File, token TokenResponse) {

    download_file := fmt.Sprintf("https://api-content.dropbox.com/1/files/dropbox/%s?access_token=%s", file.Path, token.AccessToken)

    resp, _ := http.Get(download_file)
    defer resp.Body.Close()

    filename := filepath.Base(file.Path)
    out, err := os.Create(filename)
    if err != nil {
        panic(err)
    }
    defer out.Close()

    io.Copy(out, resp.Body)
}

func main() {

    authorize_url := fmt.Sprintf("https://www.dropbox.com/1/oauth2/authorize?response_type=code&client_id=%s", app_key)

    // Get code
    fmt.Printf("1. Go to: %s\n", authorize_url)
    fmt.Println("2. Click 'Allow' (you might have to log in first)")
    fmt.Println("3. Copy the authorization code.")
    fmt.Printf("Enter the authorization code here: ")
    fmt.Scanf("%s", &code)
    // End get code

    // Get access token
    data := url.Values{}
    data.Add("code", code)
    data.Add("grant_type", "authorization_code")
    data.Add("client_id", app_key)
    data.Add("client_secret", app_secret)

    resp, _ := http.PostForm("https://api.dropbox.com/1/oauth2/token", data)
    defer resp.Body.Close()

    contents, _ := ioutil.ReadAll(resp.Body)

    var tr TokenResponse

    json.Unmarshal(contents, &tr)
    // End get access token

    // Get file list
    file_list_url := fmt.Sprintf("https://api.dropbox.com/1/metadata/dropbox/Camera Uploads?access_token=%s", tr.AccessToken)

    resp2, _ := http.Get(file_list_url)
    defer resp2.Body.Close()

    contents2, _ := ioutil.ReadAll(resp2.Body)

    var flr FileListResponse
    json.Unmarshal(contents2, &flr)
    // End get file list

    for i, file := range flr.FileList {

        download_file(file, tr)

        if i >= 2 {
            break
        }
    }
}

It doesn't work when I prefix the download_file function with the go command.

go download_file(file, tr)

Answer 1

That's because your main goroutine is exiting. You need to add a WaitGroup to wait until all the goroutines exit. For example,

var wg sync.WaitGroup
for i, file := range flr.FileList {
    wg.Add(1)

    go download_file(file, tr, wg)

    if i >= 2 {
        break
    }
}
wg.Wait()

...
func download_file(file File, token TokenResponse, wg sync.WaitGroup) {
    ...
    wg.Done()
}

Answer 2

This project might help you and others that are looking into achieving concurrency in Go:

Idea is to create a cluster in Go to execute parallel jobs. https://github.com/waqar-alamgir/mini-go-cluster

Source can be tweaked to download files concurrently.