paweld/lazarus-ccr
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fpspreadsheet: Use TBufStream as general-purpose stream if woBufStream is set in Worksheet.WritingOptions. woBufStream replaces woSaveMemory. Results in a significant speed enhancement for biff2 (64000x100 cells -> 31 sec without, 1.7 sec with woBufStream).

Browse Source 
git-svn-id: https://svn.code.sf.net/p/lazarus-ccr/svn@3337 8e941d3f-bd1b-0410-a28a-d453659cc2b4
This commit is contained in:
wp_xxyyzz 2014-07-19 13:23:12 +00:00
parent 42f55e22d7
commit 08987b52c8
8 changed files with 105 additions and 39 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
components/fpspreadsheet/examples/other/test_virtualmode.lpr
View File

@ -65,11 +65,12 @@ begin
{ These are the essential commands to activate virtual mode: }
// workbook.WritingOptions := [woVirtualMode, woSaveMemory];
workbook.WritingOptions := [woVirtualMode];
{ woSaveMemory can be omitted, but is essential for large files: it causes
writing temporaray data to a file stream instead of a memory stream.
woSaveMemory, however, considerably slows down writing of biff files. }
workbook.WritingOptions := [woVirtualMode, woBufStream];
// workbook.WritingOptions := [woVirtualMode];
{ woBufStream can be omitted, but is important for large files: it causes
writing temporary data to a buffered file stream instead of a pure
memory stream which can overflow memory. The option can slow down the
writing process a bit. }
{ Next two numbers define the size of virtual spreadsheet.
In case of a database, VirtualRowCount is the RecordCount, VirtualColCount

2
components/fpspreadsheet/fpsopendocument.pas
View File

@ -2239,7 +2239,7 @@ procedure TsSpreadOpenDocWriter.CreateStreams;
var
dir: String;
begin
if (woSaveMemory in Workbook.WritingOptions) then begin
if (woBufStream in Workbook.WritingOptions) then begin
dir := IncludeTrailingPathDelimiter(GetTempDir);
FSMeta := TFileStream.Create(GetTempFileName(dir, 'fpsM'), fmCreate+fmOpenRead);
FSSettings := TFileStream.Create(GetTempFileName(dir, 'fpsS'), fmCreate+fmOpenRead);

16
components/fpspreadsheet/fpspreadsheet.pas
View File

@ -704,10 +704,9 @@ type
@param woVirtualMode If in virtual mode date are not taken from cells
when a spreadsheet is written to file, but are
provided by means of the event OnNeedCellData.
@param woSaveMemory When this option is set temporary files are not
written to memory streams but to file streams using
temporary files. }
TsWorkbookWritingOption = (woVirtualMode, woSaveMemory);
@param woBufStream When this option is set a buffered stream is used
for writing (a memory stream swapping to disk) }
TsWorkbookWritingOption = (woVirtualMode, woBufStream);
{@@
Options considered when writing a workbook }
@ -1076,7 +1075,7 @@ function SameCellBorders(ACell1, ACell2: PCell): Boolean;
implementation
uses
Math, StrUtils, TypInfo, fpsUtils, fpsNumFormatParser, fpsFunc;
Math, StrUtils, TypInfo, fpsStreams, fpsUtils, fpsNumFormatParser, fpsFunc;
{ Translatable strings }
resourcestring
@ -5702,13 +5701,16 @@ end;
procedure TsCustomSpreadWriter.WriteToFile(const AFileName: string;
const AOverwriteExisting: Boolean = False);
var
OutputFile: TFileStream;
OutputFile: TStream;
lMode: Word;
begin
if AOverwriteExisting then lMode := fmCreate or fmOpenWrite
else lMode := fmCreate;
OutputFile := TFileStream.Create(AFileName, lMode);
if (woBufStream in Workbook.WritingOptions) then
OutputFile := TBufStream.Create(AFileName, lMode)
else
OutputFile := TFileStream.Create(AFileName, lMode);
try
WriteToStream(OutputFile);
finally

59
components/fpspreadsheet/fpsstreams.pas
View File

@ -6,7 +6,7 @@ uses
SysUtils, Classes;
const
DEFAULT_STREAM_BUFFER_SIZE = 1024; // * 1024;
DEFAULT_STREAM_BUFFER_SIZE = 1024 * 1024;
type
{ A buffered stream }
@ -18,11 +18,14 @@ type
FBufSize: Int64;
FKeepTmpFile: Boolean;
FFileName: String;
FFileMode: Word;
protected
procedure CreateFileStream;
function GetPosition: Int64; override;
function GetSize: Int64; override;
public
constructor Create(AFileName: String; AMode: Word;
ABufSize: Cardinal = DEFAULT_STREAM_BUFFER_SIZE); overload;
constructor Create(ATempFile: String; AKeepFile: Boolean = false;
ABufSize: Cardinal = DEFAULT_STREAM_BUFFER_SIZE); overload;
constructor Create(ABufSize: Cardinal = DEFAULT_STREAM_BUFFER_SIZE); overload;
@ -46,7 +49,18 @@ begin
AStream.Position := 0;
end;
{@@
Constructor of the TBufStream. Creates a memory stream and prepares everything
to create also a file stream if the streamsize exceeds ABufSize bytes.
@param ATempFile File name for the file stream. If an empty string is
used a temporary file name is created by calling GetTempFileName.
@param AKeepFile If true the stream is flushed to file when the stream is
destroyed. If false the file is deleted when the stream
is destroyed.
@param ABufSize Maximum size of the memory stream before swapping to file
starts. Value is given in bytes.
}
constructor TBufStream.Create(ATempFile: String; AKeepFile: Boolean = false;
ABufSize: Cardinal = DEFAULT_STREAM_BUFFER_SIZE);
begin
@ -60,17 +74,45 @@ begin
// The file stream is only created when needed because of possible conflicts
// of random file names.
FBufSize := ABufSize;
FFileMode := fmCreate + fmOpenRead;
end;
{@@
Constructor of the TBufStream. Creates a memory stream and prepares everything
to create also a file stream if the streamsize exceeds ABufSize bytes. The
stream created by this constructor is mainly intended to serve a temporary
purpose, it is not stored permanently to file.
@param ABufSize Maximum size of the memory stream before swapping to file
starts. Value is given in bytes.
}
constructor TBufStream.Create(ABufSize: Cardinal = DEFAULT_STREAM_BUFFER_SIZE);
begin
Create('', false, ABufSize);
end;
{@@
Constructor of the TBufStream. When swapping to file it will create a file
stream using the given file mode. This kind of BufStream is considered as a
fast replacement of TFileStream.
@param AFileName File name for the file stream. If an empty string is
used a temporary file name is created by calling GetTempFileName.
@param AMode FileMode for the file stream (fmCreate, fmOpenRead etc.)
@param ABufSize Maximum size of the memory stream before swapping to file
starts. Value is given in bytes.
}
constructor TBufStream.Create(AFileName: String; AMode: Word;
ABufSize: Cardinal = DEFAULT_STREAM_BUFFER_SIZE);
begin
Create(AFileName, true, ABufSize);
FFileMode := AMode;
end;
destructor TBufStream.Destroy;
begin
// Write current buffer content to file
FlushBuffer;
if FKeepTmpFile then FlushBuffer;
// Free streams and delete temporary file, if requested
FreeAndNil(FMemoryStream);
@ -87,7 +129,7 @@ procedure TBufStream.CreateFileStream;
begin
if FFileStream = nil then begin
if FFileName = '' then FFileName := ChangeFileExt(GetTempFileName, '.~abc');
FFileStream := TFileStream.Create(FFileName, fmCreate + fmOpenRead);
FFileStream := TFileStream.Create(FFileName, FFileMode);
end;
end;
@ -113,6 +155,8 @@ begin
Result := FFileStream.Position + FMemoryStream.Position;
end;
{ Returns the size of the stream. Both memory and file streams are considered
if needed. }
function TBufStream.GetSize: Int64;
var
n: Int64;
@ -125,6 +169,15 @@ begin
Result := Max(n, GetPosition);
end;
{@@
Reads a given number of bytes into a buffer and return the number of bytes
read. If the bytes are not in the memory stream they are read from the file
stream.
@param Buffer Buffer into which the bytes are read. Sufficient space must
have been allocated for Count bytes
@param Count Number of bytes to read from the stream
@return Number of bytes that were read from the stream.}
function TBufStream.Read(var Buffer; Count: Longint): Longint;
begin
// Case 1: All "Count" bytes are contained in memory stream

24
components/fpspreadsheet/tests/internaltests.pas
View File

@ -34,7 +34,7 @@ type
// Set up expected values:
procedure SetUp; override;
procedure TearDown; override;
procedure TestVirtualMode(AFormat: TsSpreadsheetFormat; SaveMemoryMode: Boolean);
procedure TestVirtualMode(AFormat: TsSpreadsheetFormat; ABufStreamMode: Boolean);
published
// Tests getting Excel style A1 cell locations from row/column based locations.
@ -59,10 +59,10 @@ type
procedure TestVirtualMode_BIFF8;
procedure TestVirtualMode_OOXML;
procedure TestVirtualMode_BIFF2_SaveMemory;
procedure TestVirtualMode_BIFF5_SaveMemory;
procedure TestVirtualMode_BIFF8_SaveMemory;
procedure TestVirtualMode_OOXML_SaveMemory;
procedure TestVirtualMode_BIFF2_BufStream;
procedure TestVirtualMode_BIFF5_BufStream;
procedure TestVirtualMode_BIFF8_BufStream;
procedure TestVirtualMode_OOXML_BufStream;
end;
implementation
@ -295,7 +295,7 @@ begin
end;
procedure TSpreadInternalTests.TestVirtualMode(AFormat: TsSpreadsheetFormat;
SaveMemoryMode: Boolean);
ABufStreamMode: Boolean);
var
tempFile: String;
workbook: TsWorkbook;
@ -308,8 +308,8 @@ begin
try
worksheet := workbook.AddWorksheet('VirtualMode');
workbook.WritingOptions := workbook.WritingOptions + [woVirtualMode];
if SaveMemoryMode then
workbook.WritingOptions := workbook.WritingOptions + [woSaveMemory];
if ABufStreamMode then
workbook.WritingOptions := workbook.WritingOptions + [woBufStream];
workbook.VirtualColCount := 1;
workbook.VirtualRowCount := Length(SollNumbers) + 4;
// We'll use only the first 4 SollStrings, the others cause trouble due to utf8 and formatting.
@ -368,22 +368,22 @@ begin
TestVirtualMode(sfOOXML, false);
end;
procedure TSpreadInternalTests.TestVirtualMode_BIFF2_SaveMemory;
procedure TSpreadInternalTests.TestVirtualMode_BIFF2_BufStream;
begin
TestVirtualMode(sfExcel2, True);
end;
procedure TSpreadInternalTests.TestVirtualMode_BIFF5_SaveMemory;
procedure TSpreadInternalTests.TestVirtualMode_BIFF5_BufStream;
begin
TestVirtualMode(sfExcel5, true);
end;
procedure TSpreadInternalTests.TestVirtualMode_BIFF8_SaveMemory;
procedure TSpreadInternalTests.TestVirtualMode_BIFF8_BufStream;
begin
TestVirtualMode(sfExcel8, true);
end;
procedure TSpreadInternalTests.TestVirtualMode_OOXML_SaveMemory;
procedure TSpreadInternalTests.TestVirtualMode_OOXML_BufStream;
begin
TestVirtualMode(sfOOXML, true);
end;

17
components/fpspreadsheet/xlsbiff5.pas
View File

@ -219,6 +219,9 @@ var
implementation
uses
fpsStreams;
const
{ Excel record IDs }
// see: in xlscommon
@ -330,21 +333,25 @@ end;
procedure TsSpreadBIFF5Writer.WriteToFile(const AFileName: string;
const AOverwriteExisting: Boolean);
var
MemStream: TMemoryStream;
Stream: TStream;
OutputStorage: TOLEStorage;
OLEDocument: TOLEDocument;
begin
MemStream := TMemoryStream.Create;
if (woBufStream in Workbook.WritingOptions) then begin
Stream := TBufStream.Create
end else
Stream := TMemoryStream.Create;
OutputStorage := TOLEStorage.Create;
try
WriteToStream(MemStream);
WriteToStream(Stream);
// Only one stream is necessary for any number of worksheets
OLEDocument.Stream := MemStream;
OLEDocument.Stream := Stream;
OutputStorage.WriteOLEFile(AFileName, OLEDocument, AOverwriteExisting);
finally
MemStream.Free;
Stream.Free;
OutputStorage.Free;
end;
end;

2
components/fpspreadsheet/xlsbiff8.pas
View File

@ -365,7 +365,7 @@ var
OutputStorage: TOLEStorage;
OLEDocument: TOLEDocument;
begin
if (woSaveMemory in Workbook.WritingOptions) then begin
if (woBufStream in Workbook.WritingOptions) then begin
Stream := TBufStream.Create
end else
Stream := TMemoryStream.Create;

13
components/fpspreadsheet/xlsxooxml.pas
View File

@ -883,7 +883,7 @@ begin
h0 := Workbook.GetDefaultFontSize; // Point size of default font
// Create the stream
if (woSaveMemory in Workbook.WritingOptions) then
if (woBufStream in Workbook.WritingOptions) then
FSSheets[FCurSheetNum] := TBufStream.Create(GetTempFileName('', Format('fpsSH%d', [FCurSheetNum])))
else
FSSheets[FCurSheetNum] := TMemoryStream.Create;
@ -1013,7 +1013,7 @@ end;
single xlsx file. }
procedure TsSpreadOOXMLWriter.CreateStreams;
begin
if (woSaveMemory in Workbook.WritingOptions) then begin
if (woBufStream in Workbook.WritingOptions) then begin
FSContentTypes := TBufStream.Create(GetTempFileName('', 'fpsCT'));
FSRelsRels := TBufStream.Create(GetTempFileName('', 'fpsRR'));
FSWorkbookRels := TBufStream.Create(GetTempFileName('', 'fpsWBR'));
@ -1106,14 +1106,17 @@ end;
procedure TsSpreadOOXMLWriter.WriteToFile(const AFileName: string;
const AOverwriteExisting: Boolean);
var
lStream: TFileStream;
lStream: TStream;
lMode: word;
begin
if AOverwriteExisting
then lMode := fmCreate or fmOpenWrite
else lMode := fmCreate;
lStream:=TFileStream.Create(AFileName, lMode);
if (woBufStream in Workbook.WritingOptions) then
lStream := TBufStream.Create(AFileName, lMode)
else
lStream := TFileStream.Create(AFileName, lMode);
try
WriteToStream(lStream);
finally
@ -1139,7 +1142,7 @@ begin
WriteGlobalFiles;
WriteContent;
// Stream position must be at beginning, it was moved to end during adding of xml strings.
// Stream positions must be at beginning, they were moved to end during adding of xml strings.
ResetStreams;
{ Now compress the files }