在Java中行的開頭或結尾處擊中CSV文件

Question

我正在使用此代碼來拆分和處理一個csv文件，問題是這些塊被設置在任意位置，可能在該行的開始，中間或結尾！

如何將start_loc設置為行的開頭或結尾，以便塊是完整的CSV文件而不會丟失任何數據？

public static void main(String[] args) throws IOException {
        long start = System.currentTimeMillis();

        CSVReader reader = new CSVReader(new FileReader("x_tran.csv"));
        String[] columnsNames = reader.readNext();
        reader.close();
        FileInputStream fileInputStream = new FileInputStream("x_tran.csv");
        FileChannel channel = fileInputStream.getChannel();
        long remaining_size = channel.size(); //get the total number of bytes in the file
        long chunk_size = remaining_size / 4; //file_size/threads

        //Max allocation size allowed is ~2GB
        if (chunk_size > (Integer.MAX_VALUE - 5))
        {
            chunk_size = (Integer.MAX_VALUE - 5);
        }

        //thread pool
        ExecutorService executor = Executors.newFixedThreadPool(4);

        long start_loc = 0;//file pointer
        int i = 0; //loop counter
        boolean first = true;
        while (remaining_size >= chunk_size)
        {
            //launches a new thread
            executor.execute(new FileRead(start_loc, toIntExact(chunk_size), channel, i, String.join(",", columnsNames), first));
            remaining_size = remaining_size - chunk_size;
            start_loc = start_loc + chunk_size;
            i++;
            first = false;
        }

        //load the last remaining piece
        executor.execute(new FileRead(start_loc, toIntExact(remaining_size), channel, i, String.join(",", columnsNames), first));

        //Tear Down
        executor.shutdown();

        //Wait for all threads to finish
        while (!executor.isTerminated())
        {
            //wait for infinity time
        }
        System.out.println("Finished all threads");
        fileInputStream.close();


        long finish = System.currentTimeMillis();
        System.out.println( "Time elapsed: " + (finish - start) );
    }

Answer 1

您可以讀取一次文件，然后使每個線程處理以線程數為模的行（例如，第一個線程處理行號0、4、8等）。

package ...;

import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.List;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;

public class CsvParallelReader {

    private static final int THREAD_NUMBER = 4;

    public static void main(String[] args) {


        ExecutorService executor = Executors.newFixedThreadPool(THREAD_NUMBER);


        try {
            List<String> lines = Files.readAllLines(Path.of("yourfile.csv"));

            for (int i = 0; i < THREAD_NUMBER; i++) {
                Runnable readTask = new ReadTask(i, lines);
                executor.submit(readTask);
            }
        } catch (IOException e) {
            e.printStackTrace();
        }


    }

    private static class ReadTask implements Runnable {

        private final List<String> lines;
        private int start;

        public ReadTask(int start, List<String> lines) {
            this.start = start;
            this.lines = lines;
        }

        @Override
        public void run() {
            for (int i = start; i < lines.size(); i += THREAD_NUMBER) {
                // do something with this line of data
            }
        }
    }
}