在golang中提取tarball时丢失文件

Question

I'm trying out this function to just untar a file after I've ungzip'd it, however, when it untars there are some folders missing and I can't figure out why. UnGzip works fine when I open the created tarfile via gui so that function isnt included.

func main() {
fileUrl := "https://www.clamav.net/downloads/production/clamav-0.103.1.tar.gz"
filePath := "clamav-0.103.1.tar.gz"
tempFolder := "temp"
    
err := os.Mkdir(tempFolder, 0755)
if err != nil {
    panic(err)
}

err = DownloadFile(filePath, fileUrl)
if err != nil {
    panic(err)
}
fmt.Println("Downloaded: " + fileUrl)

UnGzip(filePath,tempFolder + "/clamav.tar")
UnTar(tempFolder + "/clamav.tar",tempFolder + "/clamAV/")
//err := os.RemoveAll("tempFolder")
//if err != nil {
    //panic(err)
//}

}

func UnTar(tarball, target string) error {
reader, err := os.Open(tarball)
if err != nil {
    return err
}
defer reader.Close()
tarReader := tar.NewReader(reader)

for {
    header, err := tarReader.Next()
    if err == io.EOF {
        break
    } else if err != nil {
        return err
    }

    path := filepath.Join(target, header.Name)
    info := header.FileInfo()
    if info.IsDir() {
        if err = os.MkdirAll(path, info.Mode()); err != nil {
            return err
        }
        continue
    }

    file, err := os.OpenFile(path, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, info.Mode())
    if err != nil {
        return err
    }
    defer file.Close()
    _, err = io.Copy(file, tarReader)
    if err != nil {
        return err
    }
}
return nil

}

here's what I should get: want and here's what I have: have

Answer 1

Here is some example code:

package main

import (
   "archive/tar"
   "compress/gzip"
   "io"
   "os"
   "path"
)

func extract(source string) error {
   file, err := os.Open(source)
   if err != nil { return err }
   defer file.Close()
   gzRead, err := gzip.NewReader(file)
   if err != nil { return err }
   defer gzRead.Close()
   tarRead := tar.NewReader(gzRead)
   for {
      cur, err := tarRead.Next()
      if err == io.EOF { break } else if err != nil { return err }
      os.MkdirAll(path.Dir(cur.Name), os.ModeDir)
      switch cur.Typeflag {
      case tar.TypeReg:
         create, err := os.Create(cur.Name)
         if err != nil { return err }
         defer create.Close()
         create.ReadFrom(tarRead)
      case tar.TypeLink:
         os.Link(cur.Linkname, cur.Name)
      }
   }
   return nil
}

Usage:

package main

func main() {
   extract("clamav-0.103.1.tar.gz")
}

Answer 2

You're likely running into ulimit for the allowed number of open files per process. Run ulimit with the -a flag, and I think the default open files limit is 1024. The tarball has 2758 files.

This is because you defer the closing of the file descriptor in the for loop processing the tarReader .

To fix it, close each file as you've dealt with them:

func UnTar(tarball, target string) error {
    reader, err := os.Open(tarball)
    if err != nil {
        return err 
    }   
    defer reader.Close()
    tarReader := tar.NewReader(reader)

    for {
        header, err := tarReader.Next()
        if err == io.EOF {
            break
        } else if err != nil {
            return err 
        }

        path := filepath.Join(target, header.Name)
        info := header.FileInfo()
        if info.IsDir() {
            if err = os.MkdirAll(path, info.Mode()); err != nil {
                return err 
            }
            continue
        }

        err = processOneFile(tarReader, path, info.Mode())
        if err != nil {
            return err 
        }
    }   
    return nil 
}

func processOneFile(tarReader io.Reader, filePath string, fileMode os.FileMode) error {
    file, err := os.OpenFile(filePath, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, fileMode)
    if err != nil {
        return err 
    }   
    defer file.Close() // close error discarded
    _, err = io.Copy(file, tarReader)
    return err 
}

Answer 3

While the other answer about the ulimit is already really good, there are two things I just want to add:

1. you can decompress gzip and read the tar file at the same time instead of creating a temp file in between. You could also directly stream the file from the URL and extract it while downloading
2. Someone could probably create a malicious tar.gz file that would make your code overwrite important files using something like zipslip (especially if your program runs as root, someone could inject a file path like ../../../../etc/passwd and thus overwrite that file, maybe even edit the crontab file and execute code that way?), you should probably check for that

With that in mind, we can write a function that directly extracts from an io.Reader that also checks for any paths outside of the target directory:

// untargz decompresses a gzipped tar stream to the directory specified by target.
// Note that `file` should be closed by the caller
func untargz(file io.Reader, targetDir string) (err error) {
    gz, err := gzip.NewReader(file)
    if err != nil {
        return
    }
    // This does not close file
    defer gz.Close()

    tarReader := tar.NewReader(gz)

    for {
        header, err := tarReader.Next()
        if err == io.EOF {
            break
        } else if err != nil {
            return err
        }

        // This can be dangerous, similar to zipslip
        path := filepath.Join(targetDir, header.Name)

        // Check for ZipSlip. More Info: https://snyk.io/research/zip-slip-vulnerability#go
        if !strings.HasPrefix(path, filepath.Clean(targetDir)+string(os.PathSeparator)) {
            err = fmt.Errorf("%s: illegal file path", path)
            return err
        }

        info := header.FileInfo()
        if info.IsDir() {
            if err = os.MkdirAll(path, info.Mode()); err != nil {
                return err
            }
            continue
        }

        file, err := os.OpenFile(path, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, info.Mode())
        if err != nil {
            return err
        }

        _, err = io.Copy(file, tarReader)
        if err != nil {
            file.Close()
            return err
        }

        err = file.Close()
        if err != nil {
            return err
        }
    }

    return nil
}

It might also be beneficial to think about what would happen if a directory is declared after a file that is in that directory (because then os.Create fails), but this function doesn't handle that case.

This function could be used to directly stream to an output directory, but I'm honestly not sure if that is what you want:

func main() {
    resp, err := http.Get(`https://www.clamav.net/downloads/production/clamav-0.103.1.tar.gz`)
    if err != nil {
        panic(err)
    }
    defer resp.Body.Close()

    err = untargz(bufio.NewReader(resp.Body), "out")
    if err != nil {
        panic(err)
    }

    println("Done")
}

You can find the full file here .