如何克服大文件写入过程中的OutOfMemoryError

我面临的问题是,由于某些表的文件很大(约30GB),所以即使使用20GB的Java堆,我几乎每次都会耗尽内存。在创建文件的过程中,当文件大小超过堆大小时,即使使用最激进的GC策略之一,该过程似乎也会挂断。例如,如果文件大小为>20GB,堆大小为20GB,则一旦堆利用率达到最大堆大小,其每分钟写入2MB左右的速度就会减慢,以这种速度,将需要数月才能获得完全提取。

我正在寻找克服这个问题的方法。任何帮助都将不胜感激。

Java-JDK1.6.0\u 14

系统配置-运行在4 X Intel Xeon E7450(6核)上的RH Enterprise Linux(2.6.18)@2.39GH

内存-32GB

数据库Oracle 11g

文件连接部分代码如下:

private void runQuery(Connection conn, String query, String filePath,
        String fileName) throws SQLException, Exception {
    PreparedStatement stmt = null;
    ResultSet rs = null;
    try {
        stmt = conn.prepareStatement(query,
                ResultSet.TYPE_SCROLL_INSENSITIVE,
                ResultSet.CONCUR_READ_ONLY);
        stmt.setFetchSize(maxRecBeforWrite);
        rs = stmt.executeQuery();
        // Write query result to file
        writeDataToFile(rs, filePath + "/" + fileName, getRecordCount(
                query, conn));
    } catch (SQLException sqle) {
        sqle.printStackTrace();
    } finally {
        try {
            rs.close();
            stmt.close();
        } catch (SQLException ex) {
            throw ex;
        }
    }
}

private void writeDataToFile(ResultSet rs, String tempFile, String cnt)
        throws SQLException, Exception {
    FileOutputStream fileOut = null;
    int maxLength = 0;
    try {
        fileOut = new FileOutputStream(tempFile, true);
        FileChannel fcOut = fileOut.getChannel();

        List<TableMetaData> metaList = getMetaData(rs);
        maxLength = getMaxRecordLength(metaList);
        // Write Header
        writeHeaderRec(fileOut, maxLength);
        while (rs.next()) {
            // Now iterate on metaList and fetch all the column values.
            writeData(rs, metaList, fcOut);
        }
        // Write trailer
        writeTrailerRec(fileOut, cnt, maxLength);
    } catch (FileNotFoundException fnfe) {
        fnfe.printStackTrace();
    } catch (IOException ioe) {
        ioe.printStackTrace();
    } finally {
        try {
            fileOut.close();
        } catch (IOException ioe) {
            fileOut = null;
            throw new Exception(ioe.getMessage());
        }
    }
}

private void writeData(ResultSet rs, List<TableMetaData> metaList,
        FileChannel fcOut) throws SQLException, IOException {
    StringBuilder rec = new StringBuilder();
    String lf = "\n";
    for (TableMetaData tabMeta : metaList) {
        rec.append(getFormattedString(rs, tabMeta));
    }
    rec.append(lf);
    ByteBuffer byteBuf = ByteBuffer.wrap(rec.toString()
            .getBytes("US-ASCII"));
    fcOut.write(byteBuf);
}

private String getFormattedString(ResultSet rs, TableMetaData tabMeta)
        throws SQLException, IOException {
    String colValue = null;
    // check if it is a CLOB column
    if (tabMeta.isCLOB()) {
        // Column is a CLOB, so fetch it and retrieve first clobLimit chars.
        colValue = String.format("%-" + tabMeta.getColumnSize() + "s",
                getCLOBString(rs, tabMeta));
    } else {
        colValue = String.format("%-" + tabMeta.getColumnSize() + "s", rs
                .getString(tabMeta.getColumnName()));
    }
    return colValue;

}