How to upload large files in flutter web?

Question

There is already a great thread about how to pick files: Article

I end up with List<PlatformFile>? files List<PlatformFile>? files if I use file_picker. I passed withReadStream: true to the pickFiles method so I get a stream. Here is my code so far:

List<PlatformFile>? files = fileUploadView.result?.files;
for (PlatformFile file in files!) {


//-----add selected file with request
request.files.add(http.MultipartFile(
    "Your parameter name on server side",  file.readStream!,
    file.size,
    filename: file.name));
}

//-------Send request
var resp = await request.send();

But if I run it an error occours every few seconds:

RangeError: Array buffer allocation failed
    at new ArrayBuffer (<anonymous>)
    at new Uint8Array (<anonymous>)
    at Function._create1 (http://localhost:54372/dart_sdk.js:32192:14)
    at Function.new (http://localhost:54372/dart_sdk.js:32155:49)

Answer 1

A few weeks later I am able to post an answer: The way to go is to use a chunk uploader. This means to manually send the file in little parts. I send 99MB per request for example. There is already a basic implementation of this online: https://pub.dev/packages/chunked_uploader

You have to get a stream, this is possible with the file_picker or the drop_zone library. I used the drop_zone library because it provides the file picker and the drop zone functionality. In my code the dynamic file objects come from the drop_zone library.

Maybe you have to adjust the chunk uploader functionality depending one your backend. I use a django backend where I wrote a simple view that saves the files. In case of small files it can receive multipart requests with multiple files, in case of large files it can receive chunks and continiues to write a file if a previous chunk was received. Here some parts of my code:

Python backend:

@api_view(["POST"])
def upload(request):
    basePath = config.get("BasePath")
    
    targetFolder = os.path.join(basePath, request.data["taskId"], "input")
    if not os.path.exists(targetFolder):
        os.makedirs(targetFolder)

    for count, file in enumerate(request.FILES.getlist("Your parameter name on server side")):
        path = os.path.join(targetFolder, file.name)
        print(path)
        with open(path, 'ab') as destination:
            for chunk in file.chunks():
                destination.write(chunk)

    return HttpResponse("File(s) uploaded!")

flutter chunk uploader in my version:

import 'dart:async';
import 'dart:html';
import 'dart:math';
import 'package:dio/dio.dart';
import 'package:flutter_dropzone/flutter_dropzone.dart';
import 'package:http/http.dart' as http;

class UploadRequest {
  final Dio dio;
  final String url;
  final String method;
  final String fileKey;
  final Map<String, String>? bodyData;
  final Map<String, String>? headers;
  final CancelToken? cancelToken;
  final dynamic file;
  final Function(double)? onUploadProgress;
  late final int _maxChunkSize;
  int fileSize;
  String fileName;
  late DropzoneViewController controller;

  UploadRequest(
    this.dio, {
    required this.url,
    this.method = "POST",
    this.fileKey = "file",
    this.bodyData = const {},
    this.cancelToken,
    required this.file,
    this.onUploadProgress,
    int maxChunkSize = 1024 * 1024 * 99,
    required this.controller,
    required this.fileSize,
    required this.fileName,
    this.headers
  }) {
    _maxChunkSize = min(fileSize, maxChunkSize);
  }

  Future<Response?> upload() async {
    Response? finalResponse;
    for (int i = 0; i < _chunksCount; i++) {
      final start = _getChunkStart(i);
      print("start is $start");
      final end = _getChunkEnd(i);
      final chunkStream = _getChunkStream(start, end);
      
      
      var request = http.MultipartRequest(
        "POST",
        Uri.parse(url),
      );

      //request.headers.addAll(_getHeaders(start, end));
      request.headers.addAll(headers!);

      //-----add other fields if needed
      request.fields.addAll(bodyData!);

      request.files.add(http.MultipartFile(
        "Your parameter name on server side",
        chunkStream,
        fileSize,
        filename: fileName// + i.toString(),
        )
      );


      //-------Send request
      var resp = await request.send();

      //------Read response
      String result = await resp.stream.bytesToString();

      //-------Your response
      print(result);

      
    }
    return finalResponse;
  }

  Stream<List<int>> _getChunkStream(int start, int end) async* {
    print("reading from $start to $end");
    final reader = FileReader();
    final blob = file.slice(start, end);
    reader.readAsArrayBuffer(blob);
    await reader.onLoad.first;
    yield reader.result as List<int>;
  }


  // Updating total upload progress
  _updateProgress(int chunkIndex, int chunkCurrent, int chunkTotal) {
    int totalUploadedSize = (chunkIndex * _maxChunkSize) + chunkCurrent;
    double totalUploadProgress = totalUploadedSize / fileSize;
    this.onUploadProgress?.call(totalUploadProgress);
  }

  // Returning start byte offset of current chunk
  int _getChunkStart(int chunkIndex) => chunkIndex * _maxChunkSize;

  // Returning end byte offset of current chunk
  int _getChunkEnd(int chunkIndex) =>
      min((chunkIndex + 1) * _maxChunkSize, fileSize);

  // Returning a header map object containing Content-Range
  // https://tools.ietf.org/html/rfc7233#section-2
  Map<String, String> _getHeaders(int start, int end) {
    var header = {'Content-Range': 'bytes $start-${end - 1}/$fileSize'};
    if (headers != null) {
      header.addAll(headers!);
    }
    return header;
  }

  // Returning chunks count based on file size and maximum chunk size
  int get _chunksCount {
    var result = (fileSize / _maxChunkSize).ceil();
    return result;
  }
}

    

Upload code that decides whether to upload multiple files in one request or one file divided to many requests:

//upload the large files

Map<String, String> headers = {
  'Authorization': requester.loginToken!
};

fileUploadView.droppedFiles.sort((a, b) => b.size - a.size);

//calculate the sum of teh files:

double sumInMb = 0;
int divideBy = 1000000;

for (UploadableFile file in fileUploadView.droppedFiles) {
    sumInMb += file.size / divideBy;
}

var dio = Dio();

int uploadedAlready = 0;
for (UploadableFile file in fileUploadView.droppedFiles) {

  if (sumInMb < 99) {
    break;
  }

  var uploadRequest = UploadRequest(
    dio,
    url: requester.backendApi+ "/upload",
    file: file.file,
    controller: fileUploadView.controller!,
    fileSize: file.size,
    fileName: file.name,
    headers: headers,
    bodyData: {
      "taskId": taskId.toString(),
      "user": requester.username!,
    },
  );

  await uploadRequest.upload();

  uploadedAlready++;
  sumInMb -= file.size / divideBy;
}

if (uploadedAlready > 0) {
  fileUploadView.droppedFiles.removeRange(0, uploadedAlready);
}

print("large files uploaded");

// upload the small files

//---Create http package multipart request object
var request = http.MultipartRequest(
  "POST",
  Uri.parse(requester.backendApi+ "/upload"),
);


request.headers.addAll(headers);

//-----add other fields if needed
request.fields["taskId"] = taskId.toString();

print("adding files selected with drop zone");
for (UploadableFile file in fileUploadView.droppedFiles) {

  Stream<List<int>>? stream = fileUploadView.controller?.getFileStream(file.file);

  print("sending " + file.name);

  request.files.add(http.MultipartFile(
      "Your parameter name on server side",
      stream!,
      file.size,
      filename: file.name));
}


//-------Send request
var resp = await request.send();

//------Read response
String result = await resp.stream.bytesToString();

//-------Your response
print(result);

Hopefully this gives you a good overview how I solved the problem.