将数据从 Oracle 导出到 Excel - 性能问题
Posted
技术标签:
【中文标题】将数据从 Oracle 导出到 Excel - 性能问题【英文标题】：Export Data From Oracle to Excel - Performance issue 【发布时间】：2015-04-21 17:21:04 【问题描述】：我正在尝试通过 XML 使用 PL/SQL 代码创建 xls 文件。 我已经提到：
Create xls file using PL/SQL without going through xml
generate XLS files using PL/SQL
我提到的代码可以在以下位置找到： https://akdora.wordpress.com/2009/02/06/how-to-write-excel-via-plsql-and-save-the-file-to-a-directory/
但是该过程需要大量时间来创建 xls 文件（已经 160 分钟，过程仍在执行以导出 17000 行和 12 列）
我需要将大数据（160+ 列和 400-500 k 行）导出为 xls 格式 我将无法使用任何付费插件/包。
谁能帮助我如何提高程序的性能。
包装代码：
CREATE OR REPLACE PACKAGE pkg_excel_export IS
/**
* @author : Özay AKDORA
* @version : 1.0
*
* Name of the Application : pkg_excel_export.sql
* Creation/Modification History : 5-Jan-2009
*
* Overview of Package/Sample :Create Excel files via PL/SQL
* write the file to a directory
*
**/
PROCEDURE excel_open(l_xml_body IN OUT NOCOPY CLOB);
PROCEDURE excel_close(l_xml_body IN OUT NOCOPY CLOB);
PROCEDURE worksheet_open
(
l_xml_body IN OUT NOCOPY CLOB,
p_worksheetname IN VARCHAR2
);
PROCEDURE worksheet_close(l_xml_body IN OUT NOCOPY CLOB);
PROCEDURE row_open(l_xml_body IN OUT NOCOPY CLOB);
PROCEDURE row_close(l_xml_body IN OUT NOCOPY CLOB);
PROCEDURE cell_write
(
l_xml_body IN OUT NOCOPY CLOB,
p_content IN VARCHAR2
);
PROCEDURE excel_get
(
l_xml_body IN OUT NOCOPY CLOB,
p_filename IN VARCHAR2
);
PROCEDURE prc_write_file
(
p_filename IN VARCHAR2,
p_dir IN VARCHAR2,
p_clob IN CLOB
);
END pkg_excel_export;
/
CREATE OR REPLACE PACKAGE BODY pkg_excel_export IS
/**
* Opens the excel file
*
**/
PROCEDURE excel_open(l_xml_body IN OUT NOCOPY CLOB) IS
BEGIN
l_xml_body := '<?xml version="1.0" encoding="ISO-8859-9"?>' || chr(10) ||
'<Workbook xmlns="urn:schemas-microsoft-com:office:spreadsheet"' ||
chr(10) ||
'xmlns:o="urn:schemas-microsoft-com:office:office"' ||
chr(10) ||
'xmlns:x="urn:schemas-microsoft-com:office:excel"' ||
chr(10) ||
'xmlns:ss="urn:schemas-microsoft-com:office:spreadsheet"' ||
chr(10) ||
'xmlns:html="http://www.w3.org/TR/REC-html40">' ||
chr(10) ||
'<ExcelWorkbook xmlns="urn:schemas-microsoft-com:office:excel">' ||
chr(10) || '<WindowHeight>8580</WindowHeight>' ||
chr(10) || '<WindowWidth>15180</WindowWidth>' || chr(10) ||
'<WindowTopX>120</WindowTopX>' || chr(10) ||
'<WindowTopY>45</WindowTopY>' || chr(10) ||
'<ProtectStructure>False</ProtectStructure>' || chr(10) ||
'<ProtectWindows>False</ProtectWindows>' || chr(10) ||
'</ExcelWorkbook>' || chr(10) || '<Styles>' || chr(10) ||
'<Style ss:ID="Default" ss:Name="Normal">' || chr(10) ||
'<Alignment ss:Vertical="Bottom"/>' || chr(10) ||
'<Borders/>' || chr(10) || '<Font/>' || chr(10) ||
'<Interior/>' || chr(10) || '<NumberFormat/>' || chr(10) ||
'<Protection/>' || chr(10) || '</Style>' || chr(10) ||
'<Style ss:ID="s22">' || chr(10) ||
'<Font x:Family="Swiss" ss:Bold="1" ss:Underline="Single"/>' ||
chr(10) || '</Style>' || chr(10) || '</Styles>';
END excel_open;
/**
* Closes the excel file
*
**/
PROCEDURE excel_close(l_xml_body IN OUT NOCOPY CLOB) IS
BEGIN
l_xml_body := l_xml_body || '</Workbook>';
END excel_close;
/**
* Opens a worksheet in the Excel file.
* You may open multiple worksheets.
**/
PROCEDURE worksheet_open
(
l_xml_body IN OUT NOCOPY CLOB,
p_worksheetname IN VARCHAR2
) IS
BEGIN
--
-- Create the worksheet
--
l_xml_body := l_xml_body || '<Worksheet ss:Name="' || p_worksheetname ||
'"><Table>';
END worksheet_open;
/**
* Closes the worksheet in the Excel file.
*
**/
PROCEDURE worksheet_close(l_xml_body IN OUT NOCOPY CLOB) IS
BEGIN
l_xml_body := l_xml_body || '</Table></Worksheet>';
END worksheet_close;
/**
* Opens the row tag
*
**/
PROCEDURE row_open(l_xml_body IN OUT NOCOPY CLOB) IS
BEGIN
l_xml_body := l_xml_body || '<Row>';
END row_open;
/**
* Closes the row tag
*
**/
PROCEDURE row_close(l_xml_body IN OUT NOCOPY CLOB) IS
BEGIN
l_xml_body := l_xml_body || '</Row>' || chr(10);
END row_close;
/**
* After opening the row, we can write something the first cell
* If you want it blank, write ''
**/
PROCEDURE cell_write
(
l_xml_body IN OUT NOCOPY CLOB,
p_content IN VARCHAR2
) IS
BEGIN
l_xml_body := l_xml_body || '<Cell><Data ss:Type="String"> ' ||
p_content || ' </Data></Cell>';
END cell_write;
/**
* If you are using this package from APEX, you get download the excel file.
*
**/
PROCEDURE excel_get
(
l_xml_body IN OUT NOCOPY CLOB,
p_filename IN VARCHAR2
) IS
xx BLOB;
do NUMBER;
so NUMBER;
bc NUMBER;
lc NUMBER;
w NUMBER;
BEGIN
dbms_lob.createtemporary(xx, TRUE);
do := 1;
so := 1;
bc := dbms_lob.default_csid;
lc := dbms_lob.default_lang_ctx;
w := dbms_lob.no_warning;
dbms_lob.converttoblob(xx,
l_xml_body,
dbms_lob.lobmaxsize,
do,
so,
bc,
lc,
w);
owa_util.mime_header('application/octet', FALSE);
-- set the size so the browser knows how much to download
htp.p('Content-length: ' || dbms_lob.getlength(xx));
-- the filename will be used by the browser if the users does a save as
htp.p('Content-Disposition: attachment; filename="' || p_filename ||
'.xml' || '"');
-- close the headers
owa_util.http_header_close;
-- download the BLOB
wpg_docload.download_file(xx);
END excel_get;
/**
* Writes the Excel file to some directory with a name.
* This procedure writes the CLOB data to file
*
**/
PROCEDURE prc_write_file
(
p_filename IN VARCHAR2,
p_dir IN VARCHAR2,
p_clob IN CLOB
) IS
c_amount CONSTANT BINARY_INTEGER := 32767;
l_buffer VARCHAR2(32767);
l_chr10 PLS_INTEGER;
l_cloblen PLS_INTEGER;
l_fhandler utl_file.file_type;
l_pos PLS_INTEGER := 1;
BEGIN
l_cloblen := dbms_lob.getlength(p_clob);
l_fhandler := utl_file.fopen(p_dir, p_filename, 'W', c_amount);
WHILE l_pos < l_cloblen
LOOP
l_buffer := dbms_lob.substr(p_clob, c_amount, l_pos);
EXIT WHEN l_buffer IS NULL;
l_chr10 := instr(l_buffer, chr(10), -1);
IF l_chr10 != 0
THEN
l_buffer := substr(l_buffer, 1, l_chr10 - 1);
END IF;
utl_file.put_line(l_fhandler, l_buffer, TRUE);
l_pos := l_pos + least(length(l_buffer) + 1, c_amount);
END LOOP;
utl_file.fclose(l_fhandler);
EXCEPTION
--WE SHOULD HANDLE THE FILE EXCEPTIONS HERE!!!!!
WHEN OTHERS THEN
IF utl_file.is_open(l_fhandler)
THEN
utl_file.fclose(l_fhandler);
END IF;
RAISE;
END;
END pkg_excel_export;
耗时的程序：
CREATE OR REPLACE PROCEDURE SP_STG_BT_GENERATE_XLS_RPT(p_vinput_query IN VARCHAR2,
P_report_name IN VARCHAR2,
p_user_email IN VARCHAR2,
p_module_name IN VARCHAR2,
p_vreturncode IN OUT VARCHAR2,
p_vreturnmsg IN OUT VARCHAR2) IS
cur PLS_INTEGER := DBMS_SQL.OPEN_CURSOR;
rec_tab DBMS_SQL.desc_tab;
col_cnt PLS_INTEGER;
dum NUMBER;
Select_Statement varchar2(15000);
total NUMBER;
v_rows_ret NUMBER;
v_finalxls VARCHAR2(32767);
v_col_val VARCHAR2(32767);
v_column_name VARCHAR2(32767);
myexcelcontent CLOB;
v_sysdate varchar2(50);
BEGIN
p_vreturncode := 0;
p_vreturnmsg := '';
pkg_excel_export.excel_open(myexcelcontent);
--open a worksheet
pkg_excel_export.worksheet_open(myexcelcontent, 'test');
-- Select_Statement := 'SELECT * FROM t_stg_code where code=''100118''';
Select_Statement := p_vinput_query;
Cur := DBMS_SQL.OPEN_CURSOR;
DBMS_SQL.PARSE(Cur, Select_Statement, DBMS_SQL.NATIVE);
dum := DBMS_SQL.EXECUTE(Cur);
select dum into total from dual;
DBMS_SQL.DESCRIBE_COLUMNS(Cur, col_cnt, rec_tab);
pkg_excel_export.row_open(myexcelcontent);
FOR j in 1 .. col_cnt LOOP
/* CASE rec_tab(j).col_type
WHEN 1 THEN
DBMS_SQL.DEFINE_COLUMN(Cur, j, 1, 2000);
WHEN 2 THEN
DBMS_SQL.DEFINE_COLUMN(Cur, j, 1);
WHEN 12 THEN
DBMS_SQL.DEFINE_COLUMN(Cur, j, 1);
ELSE
DBMS_SQL.DEFINE_COLUMN(Cur, j, 1, 2000);
END CASE;*/
DBMS_SQL.DEFINE_COLUMN(Cur, j, 1, 2000);
v_column_name := upper(rec_tab(j).col_name);
v_column_name := REPLACE(v_column_name, '"', '"');
v_column_name := REPLACE(v_column_name, '''', ''');
v_column_name := REPLACE(v_column_name, '<', '<');
v_column_name := REPLACE(v_column_name, '>', '>');
v_column_name := REPLACE(v_column_name, '&', '&');
pkg_excel_export.cell_write(myexcelcontent, upper(rec_tab(j).col_name));
END LOOP;
pkg_excel_export.row_close(myexcelcontent);
-- This part outputs the HEADER
--------------------------------------------------------------
LOOP
v_rows_ret := DBMS_SQL.FETCH_ROWS(Cur);
EXIT WHEN v_rows_ret = 0;
v_finalxls := NULL;
pkg_excel_export.row_open(myexcelcontent);
FOR j in 1 .. col_cnt LOOP
CASE rec_tab(j).col_type
WHEN 1 THEN
DBMS_SQL.COLUMN_VALUE(Cur, j, v_col_val);
--v_finalxls := ltrim(v_finalxls||'|"'||v_col_val||'"','|');
v_finalxls := ltrim(v_finalxls || '|' || v_col_val, '|');
-- DBMS_OUTPUT.PUT_LINE(v_col_val);
v_col_val := REPLACE(v_col_val, '"', '"');
v_col_val := REPLACE(v_col_val, '''', ''');
v_col_val := REPLACE(v_col_val, '<', '<');
v_col_val := REPLACE(v_col_val, '>', '>');
v_col_val := REPLACE(v_col_val, '&', '&');
pkg_excel_export.cell_write(myexcelcontent, v_col_val);
WHEN 2 THEN
DBMS_SQL.COLUMN_VALUE(Cur, j, v_col_val);
v_finalxls := ltrim(v_finalxls || '|' || v_col_val, '|');
-- DBMS_OUTPUT.PUT_LINE(v_col_val);
v_col_val := REPLACE(v_col_val, '"', '"');
v_col_val := REPLACE(v_col_val, '''', ''');
v_col_val := REPLACE(v_col_val, '<', '<');
v_col_val := REPLACE(v_col_val, '>', '>');
v_col_val := REPLACE(v_col_val, '&', '&');
pkg_excel_export.cell_write(myexcelcontent, v_col_val);
/* WHEN 12 THEN
DBMS_SQL.COLUMN_VALUE(Cur, j, v_date_val);
--v_finalxls := ltrim(v_finalxls||'|'||to_char(v_date_val,'DD/MM/YYYY HH24:MI:SS'),'|');
v_finalxls := ltrim(v_finalxls || '|' ||
to_char(v_date_val, 'DD/MM/YYYY HH24:MI:SS'),
'|');
DBMS_OUTPUT.PUT_LINE(v_col_val);*/
ELSE
--v_finalxls := ltrim(v_finalxls||'|"'||v_col_val||'"','|');
v_finalxls := ltrim(v_finalxls || '|' || v_col_val, '|');
-- DBMS_OUTPUT.PUT_LINE(v_col_val);
v_col_val := REPLACE(v_col_val, '"', '"');
v_col_val := REPLACE(v_col_val, '''', ''');
v_col_val := REPLACE(v_col_val, '<', '<');
v_col_val := REPLACE(v_col_val, '>', '>');
v_col_val := REPLACE(v_col_val, '&', '&');
pkg_excel_export.cell_write(myexcelcontent, v_col_val);
END CASE;
END LOOP;
pkg_excel_export.row_close(myexcelcontent);
--DBMS_OUTPUT.PUT_LINE(v_finalxls);
-- UTL_FILE.PUT_LINE(v_outfile, v_finalxls);
END LOOP;
pkg_excel_export.worksheet_close(myexcelcontent);
--close the file
pkg_excel_export.excel_close(myexcelcontent);
--get the Time Stamp
select to_char(sysdate, 'DDMMYYYYHH24MISS') into v_sysdate from dual;
--write the file somewhere
pkg_excel_export.prc_write_file(p_filename =>p_module_name||'~'||p_user_email||'~'||P_report_name||'_'||v_sysdate||'.xls',
p_dir => 'G:\XLS\',
p_clob => myexcelcontent);
-- dbms_output.put_line(substr(myexcelcontent, 1, 10000));
END SP_STG_BT_GENERATE_XLS_RPT;
我们是否有任何可用于生成 xls 的内置 oracle 功能或免费插件。 或者如果可以优化此代码以提供更快的结果（10 万行在
【问题讨论】：
调试的第一步是弄清楚是你的代码还是 SQL 查询花费了这么长时间。能否通过 SQL*Plus 运行查询，看看返回需要多长时间？ 虽然不如逐个字符处理文件有趣，但您是否尝试过使用 Toad 导出为 Excel？您可以检查 SQL Developer（免费）是否这样做，但我对此表示怀疑（除非您将 csv 文件视为“Excel”文件） 【参考方案1】：基本的 XML 生成
此示例使用 DBMS_XMLQUERY.GETXML 函数从查询中返回 XML。这是一种礼貌功能，它执行所有必要的操作，但相当不灵活。
SET SERVEROUTPUT ON
DECLARE
v_file UTL_FILE.file_type;
v_xml CLOB;
v_more BOOLEAN := TRUE;
BEGIN
-- Create XML document from query.
v_xml := DBMS_XMLQUERY.getxml('SELECT table_name, tablespace_name FROM user_tables WHERE rownum & 6');
-- Output XML document to file.
v_file := UTL_FILE.fopen('C:\Development\XML\', 'test1.xml', 'w');
WHILE v_more LOOP
UTL_FILE.put(v_file, Substr(v_xml, 1, 32767));
IF LENGTH(v_xml) > 32767 THEN
v_xml := SUBSTR(v_xml, 32768);
ELSE
v_more := FALSE;
END IF;
END LOOP;
UTL_FILE.fclose(v_file);
EXCEPTION
WHEN OTHERS THEN
DBMS_OUTPUT.put_line(Substr(SQLERRM,1,255));
UTL_FILE.fclose(v_file);
END;
/
【讨论】：
【参考方案2】：我写了你使用的包。使用“||”将字符串附加到 CLOB降低性能。我意识到，在我将它与像你这样的庞大数据一起使用之后。请使用 CLOB 的附加功能更改功能（尤其是 PROCEDURE cell_write）。
查看示例如何使用它： Clob Append
【讨论】：
以上是关于将数据从 Oracle 导出到 Excel - 性能问题的主要内容，如果未能解决你的问题，请参考以下文章