Hibernate处理大数据量的问题初探

?

解决数据量大的问题我觉得最主要的有两点：1、即使清理缓存在处理大数据量时，会有大量的数据缓冲保存在Session的一级缓存中，这缓存大太时会严重显示性能，所以在使用Hibernate处理大数据量的，可以使用session.clear()或者session. evict(Object) 在处理过程中，清除全部的缓存或者清除某个对象。2、减少与数据库的交互次数对于大数据量新增、修改、删除操作或者是对大数据量的查询，与数据库的交互次数是决定处理时间的最重要因素，减少交互的次数是提升效率的最好途径。 

在Hibernate的配置文件中有一项hibernate.jdbc.batch_sizeHibernate提供了配置参数来设置JDBC的批处理操作的大小。但并不是说这个数值越大就越好。一般设为30，50就够了，至于这个参数的性能问题，找了许久也没人能说出个大概。这里不做讨论。如果启用了二级缓存，从机制上讲Hibernate为了维护二级缓存，我们在做插入、更新、删除操作时，Hibernate都会往二级缓存充入相应的数据。性能上就会有很大损失，所以笔者建议在批处理情况下禁用二级缓存。将use_second_level_cache=false;接着就熟悉的代码坐下解释。public ModelAndView saveImport(HttpServletRequest request????,HttpServletResponse response) {???String fileName = StringUtils.varFormat(request.getParameter("fileName"));???boolean successful = true;???if (fileName.equals("")) {????request.setAttribute("mesage", this.MSG_NO_DATA_FOUND);????return new ModelAndView(this.failView);???}???Map formatMap = new HashMap();???int i=0;???formatMap.put(i++, "customerName");???formatMap.put(i++, "customerSex");???formatMap.put(i++, "phoneNo");???formatMap.put(i++, "categoryName");???formatMap.put(i++, "cardType");???formatMap.put(i++, "cardId");???//添加2010-03-31???formatMap.put(i++,"customerCorp");???formatMap.put(i++,"homeAddr");???formatMap.put(i++,"customerJoy");???List data = ExcelUtils.importExcel(fileName, formatMap, true);//包含标题???if (data.size() == 0) {????request.setAttribute("mesage", this.MSG_NO_DATA_FOUND);????return new ModelAndView(this.failView);???}???//定义手机集合，判断导入的数据是否重复???StringBuffer phoneNos=new StringBuffer();???//定义客户类型集合，避免每次查询，节约时间???StringBuffer types=new StringBuffer();???//开启事务???Session session = customerDao.openSession();???Transaction tx = session.beginTransaction();???tx.begin();???//定义循环因素???int num=1;???//获得公司部门Id与实体，节省时间???int corpSid=this.staff.getCompany().getCorpSid();???String deptSid=this.staff.getDepartment().getDepartmentSid();???Company company=this.staff.getCompany();???Department department=this.staff.getDepartment();???//导入数据??? try{???? //将客户类型从数据库中拿出来放入List中，以后有用到客户类型表的时候就不用在从数据库中去取了，减少数据库访问的次数。这点很重要。???? CustomerCategory category = new CustomerCategory();???? List<CustomerCategory> categoryList=customerCategoryDao.createQuery("from CustomerCategory t WHERE " +???? ??"t.department.departmentSid='"+deptSid+"' and t.company.corpSid="+corpSid).list();???for (Map map : (List<Map>)data) {????? String customerName = StringUtils.varFormat(String.valueOf(map.get("customerName")));????? String categoryName = StringUtils.varFormat(String.valueOf(map.get("categoryName")));????? String sex = StringUtils.varFormat(String.valueOf(map.get("customerSex")));?????String phoneNo = StringUtils.varFormat(String.valueOf(map.get("phoneNo")));?????String cardType = StringUtils.varFormat(String.valueOf(map.get("cardType")));?????String cardId = StringUtils.varFormat(String.valueOf(map.get("cardId")));?????String customerCorp=StringUtils.varFormat(String.valueOf(map.get("customerCorp")));?????String homeAddr=StringUtils.varFormat(String.valueOf(map.get("homeAddr")));?????String customerJoy=StringUtils.varFormat(String.valueOf(map.get("customerJoy")));?????int index=phoneNos.indexOf(phoneNo);?????//判断导入的手机号码自身有没有重复?????if(index!=-1){??????continue;?????}?????phoneNos.append(phoneNo+",");?????//判断手机号码长度?????if((!("".equals(phoneNo)))&& (phoneNo.length()==11||phoneNo.length()==7||phoneNo.length()==8)) {?????String sql;?????sql="select * from customer? where phone_no='"+phoneNo+"' and dept_sid_fk='"+deptSid+"' and corp_sid_fk="+corpSid;?????//在此我用了session.createSQLQuery(sql).list(),而不是customerDao.find(hql),避免了数据从数据库中查出来以后，封装成对象的过程。不过这样会带来一个数据对象转换的问题，可以对比下边的一处代码，那个地方我还是用了hql.?????List<Customer> customers=session.createSQLQuery(sql).list();?????if(customers.size()==0){??????Customer customer = new Customer();??????customer.setCustomerSex(sex);??????customer.setPhoneNo(phoneNo);??????customer.setCompany(company);??????customer.setDepartment(department);??????customer.setCardType(cardType);??????customer.setCardId(cardId);??????customer.setSmsConfirmState("N");??????customer.setCustomerCorp(customerCorp);??????customer.setHomeAddr(homeAddr);??????customer.setCustomerJoy(customerJoy);??????if ("".equals(customerName)) {???????customerName = "--UNNAMED--";??????}??????customer.setCustomerName(customerName);??????if (!("".equals(categoryName))) {//指定分类???????//这里就是我为什么要定义一个StringBuffer类型的types的原因，当前面已经执行过相同客户类型的操作时，就可以完全的跳过index_type==-1中的操作，减少数据库的访问。???????int index_type=types.indexOf(categoryName);???????if(index_type==-1){????????//这里用了session.createQuery（hql）.list().而不是session.createSQLQuery（sql）.list(),因为这样会在category=categoryList.get(m);时报类型转换异常。????????List<CustomerCategory> categorys = session.createQuery("from CustomerCategory t WHERE t.department.departmentSid='"+deptSid+"' AND t.categoryName='"+StringUtils.sqlFormat(categoryName)+"'").list();????????if (categorys.size() == 0) {?????????category = new CustomerCategory();?????????category.setCategoryName(categoryName);?????????category.setCompany(company);?????????category.setDepartment(department);?????????category.setCreateDate(new Date());?????????customerCategoryDao.save(category);????????}else{?????????category=categorys.get(0);????????}????????types.append(categoryName+",");???????}else{????????for(int m=0;m<categoryList.size();m++){?????????if(categoryList.get(m).getCategoryName().equals(categoryName)){??????????category=categoryList.get(m);?????????}????????}???????}???????//前边的if...else都是为了得到category对象。???????customer.setCustomerCategory(category);??????}??????customer.setCreateDate(new Date());??????session.save(customer);??????//此处很重要，这就是在前便提到的及时清除session中的缓存的地方，当数量达到40时及时清除，和batch_size数值对应。??????if(num%40==0){???????session.flush();???????session.clear();???????? }??????? }?????}????? num++;????}??? }catch(Exception e){????? e.printStackTrace();????? tx.rollback();????? successful=false;??? }??? if (successful) {?????tx.commit();?????session.close();?????request.setAttribute("message", this.MSG_SUCCESS);?????request.setAttribute("redirectUrl", this.requestUri+".do?method=list");?????return new ModelAndView(this.successView);????} else {?????session.close();?????request.setAttribute("message", this.MSG_FAIL);?????return new ModelAndView(this.failView);????}?? }

?