Fork/join реализует пакетный импорт и дедупликацию Excel

package com.itbbs.service;


import com.itbbs.ArrayListUtil;
import com.itbbs.DataUtil;
import com.itbbs.pojo.Student;
import com.itbbs.task.DistinctTask;

import java.util.ArrayList;
import java.util.Comparator;
import java.util.List;
import java.util.concurrent.ForkJoinPool;
import java.util.stream.Collectors;

/**
 * 学生操作service
 * tjx
 */
public class StudentService {


    /**
     * 批量导入学生
     */
    public void importStudent(List<Student> dbData,List<Student> excelData){

        // excel自身去重
        excelData = excelData.stream()
                .filter(ArrayListUtil.distinctByKey(Student::getUsername)) //根据username 去重
                .collect(Collectors.toList());
        //不重复数据
        List<Student> repetitionData = new ArrayList<>();
        long s = System.currentTimeMillis();
        //遍历所以excel数据
        for (Student data : excelData) {
            String username = data.getUsername();
            //判断是否存在于 dbData中
            if (!ArrayListUtil.isInclude(dbData, username)) {
                //如果不存在则添加到不重复集合中
                repetitionData.add(data);
            }
        }

        long e = System.currentTimeMillis();
        System.out.println("筛选耗时:"+(e-s)+"ms");
        //在数据库不重复的数据里面 筛选出不重复数据
        repetitionData =repetitionData.stream()
                .sorted(Comparator.comparing(Student::getUsername))//根据username 排序
                .collect(Collectors.toList());

//        repetitionData.forEach(p-> System.out.println(p.getUsername()));

    }


    /**
     * 批量导入学生
     */
    public void importStudent2(List<Student> dbData,List<Student> excelData){
        // 自身去重
        excelData = excelData.stream()
                .filter(ArrayListUtil.distinctByKey(Student::getUsername)) //根据username 去重
                .collect(Collectors.toList());

        long s = System.currentTimeMillis();
        //获取不重复数据
        ForkJoinPool fjp = new ForkJoinPool();
        DistinctTask task  = new DistinctTask(0,excelData.size(),dbData,excelData);
        List<Student> repetitionData = fjp.invoke(task);
        long e = System.currentTimeMillis();
        System.out.println("筛选耗时:"+(e-s)+"ms");
        //在数据库不重复的数据里面 筛选出不重复数据
        repetitionData =repetitionData.stream()
                .sorted(Comparator.comparing(Student::getUsername))//根据username 排序
                .collect(Collectors.toList());
//        repetitionData.forEach(p-> System.out.println(p.getUsername()));
    }
    public static void main(String[] args) {
        // 模拟获取数据库
        List<Student> dbData = DataUtil.getDbData();

        // 模拟获取excel数据
        List<Student> excelData = DataUtil.getExcelData();
        new StudentService().importStudent(dbData,excelData);
        new StudentService().importStudent2(dbData,excelData);
    }
}

package com.itbbs.task;

import com.itbbs.ArrayListUtil;
import com.itbbs.pojo.Student;

import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.RecursiveTask;

public class DistinctTask extends RecursiveTask<List<Student>> {

    //单个任务处理数据
    private static final int THRESHOLD_NUM = 5000;
    //下标
    private int start, end;

    //需要处理的数据
    private List<Student> dbData;
    private List<Student> excelData;

    public DistinctTask(int start, int end, List<Student> dbData, List<Student> excelData) {
        this.start = start;
        this.end = end;
        this.dbData = dbData;
        this.excelData = excelData;
    }

    @Override
    protected List<Student> compute() {
        //获取当前下标下的数据
        excelData = excelData.subList(start,end);
        //获取需要计算的数据量
        int size = excelData.size();
        if(size<=THRESHOLD_NUM){
            //计算
            List<Student> repetitionData = new ArrayList<>();
            //遍历所以excel数据
            for (Student data : excelData) {
                String username = data.getUsername();
                //判断是否存在于 dbData中
                if (!ArrayListUtil.isInclude(dbData, username)) {
                    //如果不存在则添加到不重复集合中
                    repetitionData.add(data);
                }
            }
            return repetitionData;





        }else{
            //拆分
            int middle = (start + end) / 2;
            DistinctTask left = new DistinctTask(start,middle,dbData,excelData);
            DistinctTask right = new DistinctTask(middle+1,end,dbData,excelData);
            //执行子任务
            left.fork();
            right.fork();
            //获取子任务结果
            //join() 方法会阻塞到结果算出来
            List<Student> lResult = left.join();
            List<Student> rResult = right.join();
            //何并结果
            lResult.addAll(rResult);
            return lResult;
        }
    }
}

package com.itbbs;

import com.itbbs.pojo.Student;

import java.util.ArrayList;
import java.util.List;
import java.util.Random;

/**
 * 数据源 工具类
 * tjx
 */
public class DataUtil {


    /**
     * 模拟 获取数据库
     * @return
     */
    public static List<Student> getDbData(){
        List<Student> result = new ArrayList<Student>();
        Random random = new Random();
        for (int i = 0; i <100000 ; i++) {
            Student student = new Student();
            student.setUsername(random.nextInt(99)+"");
            result.add(student);
        }
        return result;
    }


    /**
     * 模拟 获取excel数据
     * @return
     */
    public static List<Student> getExcelData(){
        List<Student> result = new ArrayList<Student>();
        Random random = new Random();
        for (int i = 0; i <10000 ; i++) {
            Student student = new Student();
            student.setUsername(random.nextInt(100000)+"");
            result.add(student);
        }
        return result;
    }


}

package com.itbbs;

import com.itbbs.pojo.Student;
import org.apache.commons.lang3.ArrayUtils;

import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.function.Function;
import java.util.function.Predicate;

public class ArrayListUtil {

    /**	 * 去重复元素	 * @param keyExtractor	 * @param <T>	 * @return	 */	public static <T> Predicate<T> distinctByKey(Function<? super T, Object> keyExtractor) {
        Map<Object, Boolean> map = new ConcurrentHashMap<>();		return t -> map.putIfAbsent(keyExtractor.apply(t), Boolean.TRUE) == null;
    }

    /**
     * 判断 list 是否包含 targetValue
     * @param list
     * @param targetValue
     * @return
     */
    public static boolean isInclude(List<Student> list, String targetValue){
        return  ArrayUtils.contains(list.toArray(),targetValue);
    }
}

package com.itbbs.pojo;

public class Student {

    private String username;

    private String password;

    public String getUsername() {
        return username;
    }

    public void setUsername(String username) {
        this.username = username;
    }

    public String getPassword() {
        return password;
    }

    public void setPassword(String password) {
        this.password = password;
    }

    @Override
    public boolean equals(Object obj) {
        if (obj instanceof Student) {
            Student student = (Student) obj;
            return (username.equals(student.username));
        }
        return super.equals(obj);
    }

    @Override
    public int hashCode() {
        Student student = (Student) this;
        return student.username.hashCode();
    }
}