Dbの問題へのデータ転送、Javaアプリケーションからデータベースへのデータ転送速度を上げる方法は？

Question

私のアプリケーションでは、データをファイルから取り出してデータベースに転送しています。私は400 000のレコードを持っています。最初に、それは非常にゆっくりと更新された後、データを10,000レコードまで高速で転送します。データをdbに転送するパフォーマンスを向上させるには？

gcに問題がありますか？

これは私のコードです：複数のレコードに1つのバッチ処理を実行

package com.fileupload; 
import java.io.File; 
import java.io.FileInputStream; 
import java.io.IOException; 
import java.sql.Connection; 
import java.sql.DriverManager; 
import java.sql.PreparedStatement; 
import java.sql.ResultSet; 
import java.sql.SQLException; 
import java.sql.Statement; 
import java.util.Iterator; 

import javax.servlet.ServletException; 
import javax.servlet.http.HttpServlet; 
import javax.servlet.http.HttpServletRequest; 
import javax.servlet.http.HttpServletResponse; 
import javax.servlet.http.HttpSession; 
import javax.swing.text.ZoneView; 

import org.apache.poi.openxml4j.opc.OPCPackage; 
import org.apache.poi.ss.usermodel.Cell; 
import org.apache.poi.ss.usermodel.Row; 
import org.apache.poi.xssf.usermodel.XSSFSheet; 
import org.apache.poi.xssf.usermodel.XSSFWorkbook; 
import java.io.InputStream; 
import java.util.Iterator; 

import org.apache.poi.xssf.eventusermodel.XSSFReader; 
import org.apache.poi.xssf.model.SharedStringsTable; 
import org.apache.poi.xssf.usermodel.XSSFRichTextString; 
import org.apache.poi.openxml4j.opc.OPCPackage; 
import org.xml.sax.Attributes; 
import org.xml.sax.ContentHandler; 
import org.xml.sax.InputSource; 
import org.xml.sax.SAXException; 
import org.xml.sax.XMLReader; 
import org.xml.sax.helpers.DefaultHandler; 
import org.xml.sax.helpers.XMLReaderFactory; 

public class SendDataToDb extends HttpServlet{ 
PreparedStatement ps = null; 
HttpSession hs; 
Connection con1; 
@Override 
public void doPost(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException { 

    hs = request.getSession(false); 
    try { 
     Class.forName("com.mysql.jdbc.Driver"); 
     con1 = DriverManager.getConnection("jdbc:mysql://localhost:3306/xlsx","root","Inf123#"); 
     ps = con1.prepareStatement("INSERT INTO userdetails(ID, NAME, AGE, GENDER,ADDRESS, ZONEID, LOCATION) VALUES(?, ?, ?, ?, ?, ?, ?)"); 

    } catch (ClassNotFoundException e1) { 
     e1.printStackTrace(); 
    } catch (SQLException e) { 
     e.printStackTrace(); 
    } 

    try { 
     processOneSheet("C:/Users/Penchalaiah/Desktop/New folder/"+hs.getAttribute("filename1")); 
     System.out.println("clossing the connnection"); 
     ps.close(); 
     con1.close(); 
    } catch (Exception e) { 
     e.printStackTrace(); 
    } 

} 

public void processOneSheet(String filename) throws Exception { 
    OPCPackage pkg = OPCPackage.open(filename); 
    XSSFReader r = new XSSFReader(pkg); 
    SharedStringsTable sst = r.getSharedStringsTable(); 

    XMLReader parser = fetchSheetParser(sst); 

    // To look up the Sheet Name/Sheet Order/rID, 
    // you need to process the core Workbook stream. 
    // Normally it's of the form rId# or rSheet# 
    InputStream sheet2 = r.getSheet("rId2"); 
    InputSource sheetSource = new InputSource(sheet2); 
    parser.parse(sheetSource); 
    sheet2.close(); 
} 

public XMLReader fetchSheetParser(SharedStringsTable sst) throws SAXException { 
    XMLReader parser = XMLReaderFactory.createXMLReader("org.apache.xerces.parsers.SAXParser"); 
    ContentHandler handler = new SheetHandler(sst); 
    parser.setContentHandler(handler); 
    return parser; 
} 

/** 
* See org.xml.sax.helpers.DefaultHandler javadocs 
*/ 
private class SheetHandler extends DefaultHandler { 

    private SharedStringsTable sst; 
    private String lastContents; 
    private boolean nextIsString; 
    String id; 
    String names; 
    String age; 
    String gender; 
    String address; 

    int i = 1; 

    private SheetHandler(SharedStringsTable sst) { 
     this.sst = sst; 

    } 

    public void startElement(String uri, String localName, String name, 
      Attributes attributes) throws SAXException { 
     // c => cell 
     if(name.equals("c")) { 
      // Print the cell reference 
      // Figure out if the value is an index in the SST 
      String cellType = attributes.getValue("t"); 
      if(cellType != null && cellType.equals("s")) { 
       nextIsString = true; 
      } else { 
       nextIsString = false; 
      } 
     } 
     // Clear contents cache 
     lastContents = ""; 
     //System.out.println("===>"+lastContents+"<===="); 
    } 

    public void endElement(String uri, String localName, String name) 
      throws SAXException { 
     // Process the last contents as required. 
     // Do now, as characters() may be called more than once 
     if(nextIsString) { 
      int idx = Integer.parseInt(lastContents); 
      lastContents = new XSSFRichTextString(sst.getEntryAt(idx)).toString(); 
      nextIsString = false; 

     } 

     // v => contents of a cell 
     // Output after we've seen the string contents 




     if(name.equals("v")) { 
      System.out.print(lastContents+"\t"); 

      if(i == 1){ 
       id = lastContents; 

       System.out.print(lastContents+"("+i+")"); 
      } 
      if(i == 2){ 
       names = lastContents; 

       System.out.print(lastContents+"("+i+")"); 
      } 
      if(i == 3){ 
       age = lastContents; 

       System.out.print(lastContents+"("+i+")"); 
      } 
      if(i == 4){ 
       gender = lastContents; 
       System.out.print(lastContents+"("+i+")"); 
      } 
      if(i == 5){ 
       address = lastContents; 

       System.out.print(lastContents+"("+i+")"); 
       insertInToDb(id, names, age, gender, address); 
       i = 0; 
      } 

      i++; 


     } 

    } 


    public void characters(char[] ch, int start, int length) 
      throws SAXException { 
     lastContents += new String(ch, start, length); 
    } 
} 

    public void insertInToDb(String id,String names,String age, String gender,String address){ 

     try { 

      ps.setString(1, id); 
      ps.setString(2, names); 
      ps.setString(3, age); 
      ps.setString(4, gender); 
      ps.setString(5, address); 
      ps.setString(6, (String)hs.getAttribute("zoneId1")); 
      ps.setString(7, (String)hs.getAttribute("location1")); 
      ps.executeUpdate(); 
     } catch (SQLException e) { 
      e.printStackTrace(); 
     } 

    } 

} 

Answer 1

は、複数のレコードのために各INSERTクエリを実行するよりもはるかに高速です。

10000などのバッチを作成して、必要なものを作成してからバッチを実行することができます。

Connection con = null; 
     PreparedStatement pstm = null; 
     try { 
      Class.forName("driver class"); 
      con = DriverManager. 
        getConnection("connectionUrlString","password"); 
      con.setAutoCommit(false); 
      pstm = con.prepareStatement("your insert command); 
      pstm .setInt(1, 3000); //set all parameters    
      pst.addBatch(); 
      int count[] = pst.executeBatch(); 
      for(int i=1;i<=count.length;i++){ 
       System.out.println("Query "+i+" has effected "+count[i]+" records"); 
      } 
      con.commit(); 
      pst.close(); 
      con.close();