mirror of
https://gitlab.com/freepascal.org/fpc/source.git
synced 2025-04-12 21:49:43 +02:00
rtl, utils: apply patch of Inoussa:
This patch implements collation'loading at runtime. This reduce the final executable' size as the collation's data are now externaly stored. Note that It requires the external collation files to be shipped and the program to load the collations it needs using the "LoadCollation"/"RegisterCollation" procedure(s). The external collation files are produced by "cldrparser" (while producing the static files). The root collation "ducet" 's external file is produced by "unihelper". It is important to note that these files are endian specific : * collation_*_be.bco for big endian systems * collation_*_le.bco for little endian system. The root collation should at be registered, be it staticaly by using the "unicodeducet" unit or dynamicaly by making a call sush as RegisterCollation(<collation dir>,'ducet'). It is possible, in the same application, to make use of static and dynamic. git-svn-id: trunk@25295 -
This commit is contained in:
paul
parent
721d695ae4
commit
f285948fcb
@ -274,6 +274,7 @@ type
|
||||
Props : PUCA_PropItemRec;
|
||||
VariableLowLimit : Word;
|
||||
VariableHighLimit : Word;
|
||||
Dynamic : Boolean;
|
||||
public
|
||||
function IsVariable(const AWeight : PUCA_PropWeights) : Boolean; inline;
|
||||
end;
|
||||
@ -320,8 +321,13 @@ type
|
||||
function CompareSortKey(const A, B : TUCASortKey) : Integer;overload;
|
||||
function CompareSortKey(const A : TUCASortKey; const B : array of Word) : Integer;overload;
|
||||
|
||||
function RegisterCollation(const ACollation : PUCA_DataBook) : Boolean;
|
||||
function RegisterCollation(const ACollation : PUCA_DataBook) : Boolean;overload;
|
||||
function RegisterCollation(
|
||||
const ADirectory,
|
||||
ALanguage : string
|
||||
) : Boolean;overload;
|
||||
function UnregisterCollation(const AName : ansistring): Boolean;
|
||||
procedure UnregisterCollations(const AFreeDynamicCollations : Boolean);
|
||||
function FindCollation(const AName : ansistring): PUCA_DataBook;overload;
|
||||
function FindCollation(const AIndex : Integer): PUCA_DataBook;overload;
|
||||
function GetCollationCount() : Integer;
|
||||
@ -330,6 +336,29 @@ type
|
||||
const ABaseName : ansistring;
|
||||
const AChangedFields : TCollationFields
|
||||
);
|
||||
function LoadCollation(
|
||||
const AData : Pointer;
|
||||
const ADataLength : Integer
|
||||
) : PUCA_DataBook;overload;
|
||||
function LoadCollation(const AFileName : string) : PUCA_DataBook;overload;
|
||||
function LoadCollation(
|
||||
const ADirectory,
|
||||
ALanguage : string
|
||||
) : PUCA_DataBook;overload;
|
||||
procedure FreeCollation(AItem : PUCA_DataBook);
|
||||
|
||||
type
|
||||
TEndianKind = (Little, Big);
|
||||
const
|
||||
ENDIAN_SUFFIX : array[TEndianKind] of string[2] = ('le','be');
|
||||
{$IFDEF ENDIAN_LITTLE}
|
||||
ENDIAN_NATIVE = TEndianKind.Little;
|
||||
ENDIAN_NON_NATIVE = TEndianKind.Big;
|
||||
{$ENDIF ENDIAN_LITTLE}
|
||||
{$IFDEF ENDIAN_BIG}
|
||||
ENDIAN_NATIVE = TEndianKind.Big;
|
||||
ENDIAN_NON_NATIVE = TEndianKind.Little;
|
||||
{$ENDIF ENDIAN_BIG}
|
||||
|
||||
resourcestring
|
||||
SCollationNotFound = 'Collation not found : "%s".';
|
||||
@ -535,6 +564,21 @@ begin
|
||||
Result := a <= Cardinal(b);
|
||||
end;
|
||||
|
||||
type
|
||||
TBitOrder = 0..7;
|
||||
function IsBitON(const AData : Byte; const ABit : TBitOrder) : Boolean ;inline;
|
||||
begin
|
||||
Result := ( ( AData and ( 1 shl ABit ) ) <> 0 );
|
||||
end;
|
||||
|
||||
procedure SetBit(var AData : Byte; const ABit : TBitOrder; const AValue : Boolean);inline;
|
||||
begin
|
||||
if AValue then
|
||||
AData := AData or (1 shl (ABit mod 8))
|
||||
else
|
||||
AData := AData and ( not ( 1 shl ( ABit mod 8 ) ) );
|
||||
end;
|
||||
|
||||
var
|
||||
CollationTable : array of PUCA_DataBook;
|
||||
function IndexOfCollation(const AName : string) : Integer;
|
||||
@ -565,6 +609,23 @@ begin
|
||||
end;
|
||||
end;
|
||||
|
||||
function RegisterCollation(const ADirectory, ALanguage : string) : Boolean;
|
||||
var
|
||||
cl : PUCA_DataBook;
|
||||
begin
|
||||
cl := LoadCollation(ADirectory,ALanguage);
|
||||
if (cl = nil) then
|
||||
exit(False);
|
||||
try
|
||||
Result := RegisterCollation(cl);
|
||||
except
|
||||
FreeCollation(cl);
|
||||
raise;
|
||||
end;
|
||||
if not Result then
|
||||
FreeCollation(cl);
|
||||
end;
|
||||
|
||||
function UnregisterCollation(const AName : ansistring): Boolean;
|
||||
var
|
||||
i, c : Integer;
|
||||
@ -582,6 +643,21 @@ begin
|
||||
end;
|
||||
end;
|
||||
|
||||
procedure UnregisterCollations(const AFreeDynamicCollations : Boolean);
|
||||
var
|
||||
i : Integer;
|
||||
cl : PUCA_DataBook;
|
||||
begin
|
||||
for i := Low(CollationTable) to High(CollationTable) do begin
|
||||
if CollationTable[i].Dynamic then begin
|
||||
cl := CollationTable[i];
|
||||
CollationTable[i] := nil;
|
||||
FreeCollation(cl);
|
||||
end;
|
||||
end;
|
||||
SetLength(CollationTable,0);
|
||||
end;
|
||||
|
||||
function FindCollation(const AName : ansistring): PUCA_DataBook;overload;
|
||||
var
|
||||
i : Integer;
|
||||
@ -632,6 +708,190 @@ begin
|
||||
p^.VariableLowLimit := base^.VariableHighLimit;
|
||||
end;
|
||||
|
||||
type
|
||||
TSerializedCollationHeader = packed record
|
||||
Base : TCollationName;
|
||||
Version : TCollationName;
|
||||
CollationName : TCollationName;
|
||||
VariableWeight : Byte;
|
||||
Backwards : Byte;
|
||||
BMP_Table1Length : DWord;
|
||||
BMP_Table2Length : DWord;
|
||||
OBMP_Table1Length : DWord;
|
||||
OBMP_Table2Length : DWord;
|
||||
PropCount : DWord;
|
||||
VariableLowLimit : Word;
|
||||
VariableHighLimit : Word;
|
||||
ChangedFields : Byte;
|
||||
end;
|
||||
PSerializedCollationHeader = ^TSerializedCollationHeader;
|
||||
|
||||
procedure FreeCollation(AItem : PUCA_DataBook);
|
||||
var
|
||||
h : PSerializedCollationHeader;
|
||||
begin
|
||||
if (AItem = nil) or not(AItem^.Dynamic) then
|
||||
exit;
|
||||
h := PSerializedCollationHeader(PtrUInt(AItem) + SizeOf(TUCA_DataBook));
|
||||
if (AItem^.BMP_Table1 <> nil) then
|
||||
FreeMem(AItem^.BMP_Table1,h^.BMP_Table1Length);
|
||||
if (AItem^.BMP_Table2 <> nil) then
|
||||
FreeMem(AItem^.BMP_Table2,h^.BMP_Table2Length);
|
||||
if (AItem^.OBMP_Table1 <> nil) then
|
||||
FreeMem(AItem^.OBMP_Table1,h^.OBMP_Table1Length);
|
||||
if (AItem^.OBMP_Table2 <> nil) then
|
||||
FreeMem(AItem^.OBMP_Table2,h^.OBMP_Table2Length);
|
||||
if (AItem^.Props <> nil) then
|
||||
FreeMem(AItem^.Props,h^.PropCount);
|
||||
FreeMem(AItem,(SizeOf(TUCA_DataBook)+SizeOf(TSerializedCollationHeader)));
|
||||
end;
|
||||
|
||||
function LoadCollation(
|
||||
const AData : Pointer;
|
||||
const ADataLength : Integer
|
||||
) : PUCA_DataBook;
|
||||
var
|
||||
dataPointer : PByte;
|
||||
readedLength : LongInt;
|
||||
|
||||
function ReadBuffer(ADest : Pointer; ALength : LongInt) : Boolean;
|
||||
begin
|
||||
Result := (readedLength + ALength) <= ADataLength;
|
||||
if not result then
|
||||
exit;
|
||||
Move(dataPointer^,ADest^,ALength);
|
||||
Inc(dataPointer,ALength);
|
||||
readedLength := readedLength + ALength;
|
||||
end;
|
||||
|
||||
var
|
||||
r : PUCA_DataBook;
|
||||
h : PSerializedCollationHeader;
|
||||
cfs : TCollationFields;
|
||||
i : Integer;
|
||||
baseName : TCollationName;
|
||||
begin
|
||||
readedLength := 0;
|
||||
dataPointer := AData;
|
||||
r := AllocMem((SizeOf(TUCA_DataBook)+SizeOf(TSerializedCollationHeader)));
|
||||
try
|
||||
h := PSerializedCollationHeader(PtrUInt(r) + SizeOf(TUCA_DataBook));
|
||||
if not ReadBuffer(h,SizeOf(TSerializedCollationHeader)) then
|
||||
exit;
|
||||
r^.Version := h^.Version;
|
||||
r^.CollationName := h^.CollationName;
|
||||
r^.VariableWeight := TUCA_VariableKind(h^.VariableWeight);
|
||||
r^.Backwards[0] := IsBitON(h^.Backwards,0);
|
||||
r^.Backwards[1] := IsBitON(h^.Backwards,1);
|
||||
r^.Backwards[2] := IsBitON(h^.Backwards,2);
|
||||
r^.Backwards[3] := IsBitON(h^.Backwards,3);
|
||||
if (h^.BMP_Table1Length > 0) then begin
|
||||
r^.BMP_Table1 := GetMem(h^.BMP_Table1Length);
|
||||
if not ReadBuffer(r^.BMP_Table1,h^.BMP_Table1Length) then
|
||||
exit;
|
||||
end;
|
||||
if (h^.BMP_Table2Length > 0) then begin
|
||||
r^.BMP_Table2 := GetMem(h^.BMP_Table2Length);
|
||||
if not ReadBuffer(r^.BMP_Table2,h^.BMP_Table2Length) then
|
||||
exit;
|
||||
end;
|
||||
if (h^.OBMP_Table1Length > 0) then begin
|
||||
r^.OBMP_Table1 := GetMem(h^.OBMP_Table1Length);
|
||||
if not ReadBuffer(r^.OBMP_Table1,h^.OBMP_Table1Length) then
|
||||
exit;
|
||||
end;
|
||||
if (h^.OBMP_Table2Length > 0) then begin
|
||||
r^.OBMP_Table2 := GetMem(h^.OBMP_Table2Length);
|
||||
if not ReadBuffer(r^.OBMP_Table2,h^.OBMP_Table2Length) then
|
||||
exit;
|
||||
end;
|
||||
r^.PropCount := h^.PropCount;
|
||||
if (h^.PropCount > 0) then begin
|
||||
r^.Props := GetMem(h^.PropCount);
|
||||
if not ReadBuffer(r^.Props,h^.PropCount) then
|
||||
exit;
|
||||
end;
|
||||
r^.VariableLowLimit := h^.VariableLowLimit;
|
||||
r^.VariableHighLimit := h^.VariableHighLimit;
|
||||
|
||||
cfs := [];
|
||||
for i := Ord(Low(TCollationField)) to Ord(High(TCollationField)) do begin
|
||||
if IsBitON(h^.ChangedFields,i) then
|
||||
cfs := cfs + [TCollationField(i)];
|
||||
end;
|
||||
if (h^.Base <> '') then
|
||||
baseName := h^.Base
|
||||
else if (h^.CollationName <> ROOT_COLLATION_NAME) then
|
||||
baseName := ROOT_COLLATION_NAME
|
||||
else
|
||||
baseName := '';
|
||||
if (baseName <> '') then
|
||||
PrepareCollation(r,baseName,cfs);
|
||||
r^.Dynamic := True;
|
||||
Result := r;
|
||||
except
|
||||
FreeCollation(r);
|
||||
raise;
|
||||
end;
|
||||
end;
|
||||
|
||||
{$PUSH}
|
||||
function LoadCollation(const AFileName : string) : PUCA_DataBook;
|
||||
const
|
||||
BLOCK_SIZE = 16*1024;
|
||||
var
|
||||
f : File of Byte;
|
||||
locSize, locReaded, c : LongInt;
|
||||
locBuffer : PByte;
|
||||
locBlockSize : LongInt;
|
||||
begin
|
||||
Result := nil;
|
||||
{$I-}
|
||||
if (AFileName = '') then
|
||||
exit;
|
||||
Assign(f,AFileName);
|
||||
Reset(f);
|
||||
try
|
||||
if (IOResult <> 0) then
|
||||
exit;
|
||||
locSize := FileSize(f);
|
||||
if (locSize < SizeOf(TSerializedCollationHeader)) then
|
||||
exit;
|
||||
locBuffer := GetMem(locSize);
|
||||
try
|
||||
locBlockSize := BLOCK_SIZE;
|
||||
locReaded := 0;
|
||||
while (locReaded < locSize) do begin
|
||||
if (locBlockSize > (locSize-locReaded)) then
|
||||
locBlockSize := locSize-locReaded;
|
||||
BlockRead(f,locBuffer[locReaded],locBlockSize,c);
|
||||
if (IOResult <> 0) or (c <= 0) then
|
||||
exit;
|
||||
locReaded := locReaded + c;
|
||||
end;
|
||||
Result := LoadCollation(locBuffer,locSize);
|
||||
finally
|
||||
FreeMem(locBuffer,locSize);
|
||||
end;
|
||||
finally
|
||||
Close(f);
|
||||
end;
|
||||
end;
|
||||
{$POP}
|
||||
|
||||
function LoadCollation(const ADirectory, ALanguage : string) : PUCA_DataBook;
|
||||
var
|
||||
fileName : string;
|
||||
begin
|
||||
fileName := ADirectory;
|
||||
if (fileName <> '') then begin
|
||||
if (fileName[Length(fileName)] <> DirectorySeparator) then
|
||||
fileName := fileName + DirectorySeparator;
|
||||
end;
|
||||
fileName := fileName + 'collation_' + ALanguage + '_' + ENDIAN_SUFFIX[ENDIAN_NATIVE] + '.bco';
|
||||
Result := LoadCollation(fileName);
|
||||
end;
|
||||
|
||||
{$INCLUDE unicodedata.inc}
|
||||
{$IFDEF ENDIAN_LITTLE}
|
||||
{$INCLUDE unicodedata_le.inc}
|
||||
@ -1026,21 +1286,6 @@ begin
|
||||
end;
|
||||
end;
|
||||
|
||||
type
|
||||
TBitOrder = 0..7;
|
||||
function IsBitON(const AData : Byte; const ABit : TBitOrder) : Boolean ;inline;
|
||||
begin
|
||||
Result := ( ( AData and ( 1 shl ABit ) ) <> 0 );
|
||||
end;
|
||||
|
||||
procedure SetBit(var AData : Byte; const ABit : TBitOrder; const AValue : Boolean);inline;
|
||||
begin
|
||||
if AValue then
|
||||
AData := AData or (1 shl (ABit mod 8))
|
||||
else
|
||||
AData := AData and ( not ( 1 shl ( ABit mod 8 ) ) );
|
||||
end;
|
||||
|
||||
{ TUCA_PropItemContextTreeNodeRec }
|
||||
|
||||
function TUCA_PropItemContextTreeNodeRec.GetLeftNode: PUCA_PropItemContextTreeNodeRec;
|
||||
|
||||
@ -208,14 +208,16 @@ type
|
||||
) : Integer;
|
||||
function FindCollationDefaultItemName(ACollation : TCldrCollation) : string;
|
||||
procedure GenerateCdlrCollation(
|
||||
ACollation : TCldrCollation;
|
||||
AItemName : string;
|
||||
AStoreName : string;
|
||||
ACollation : TCldrCollation;
|
||||
AItemName : string;
|
||||
AStoreName : string;
|
||||
AStream,
|
||||
ANativeEndianStream,
|
||||
AOtherEndianStream : TStream;
|
||||
ARootChars : TOrderedCharacters;
|
||||
ARootWeigths : TUCA_LineRecArray
|
||||
AOtherEndianStream,
|
||||
ABinaryNativeEndianStream,
|
||||
ABinaryOtherEndianStream : TStream;
|
||||
ARootChars : TOrderedCharacters;
|
||||
ARootWeigths : TUCA_LineRecArray
|
||||
);
|
||||
|
||||
procedure GenerateUCA_CLDR_Head(
|
||||
@ -1635,14 +1637,16 @@ begin
|
||||
end;
|
||||
|
||||
procedure GenerateCdlrCollation(
|
||||
ACollation : TCldrCollation;
|
||||
AItemName : string;
|
||||
AStoreName : string;
|
||||
ACollation : TCldrCollation;
|
||||
AItemName : string;
|
||||
AStoreName : string;
|
||||
AStream,
|
||||
ANativeEndianStream,
|
||||
AOtherEndianStream : TStream;
|
||||
ARootChars : TOrderedCharacters;
|
||||
ARootWeigths : TUCA_LineRecArray
|
||||
AOtherEndianStream,
|
||||
ABinaryNativeEndianStream,
|
||||
ABinaryOtherEndianStream : TStream;
|
||||
ARootChars : TOrderedCharacters;
|
||||
ARootWeigths : TUCA_LineRecArray
|
||||
);
|
||||
|
||||
procedure AddLine(const ALine : ansistring; ADestStream : TStream);
|
||||
@ -1665,6 +1669,8 @@ var
|
||||
ucaoSecondTable : TucaOBmpSecondTable;
|
||||
locHasProps : Boolean;
|
||||
s : string;
|
||||
serializedHeader : TSerializedCollationHeader;
|
||||
e : TCollationField;
|
||||
begin
|
||||
locItem := ACollation.Find(AItemName);
|
||||
if (locItem = nil) then
|
||||
@ -1707,6 +1713,43 @@ begin
|
||||
AddLine('{$endif FPC_LITTLE_ENDIAN}',AStream);
|
||||
end;
|
||||
GenerateUCA_CLDR_Registration(AStream,@locUcaBook);
|
||||
|
||||
FillChar(serializedHeader,SizeOf(TSerializedCollationHeader),0);
|
||||
serializedHeader.Base := locItem.Base;
|
||||
serializedHeader.Version := ACollation.Version;
|
||||
serializedHeader.CollationName := ACollation.Language;
|
||||
serializedHeader.VariableWeight := Ord(locUcaBook.VariableWeight);
|
||||
SetBit(serializedHeader.Backwards,0,locUcaBook.Backwards[0]);
|
||||
SetBit(serializedHeader.Backwards,1,locUcaBook.Backwards[1]);
|
||||
SetBit(serializedHeader.Backwards,2,locUcaBook.Backwards[2]);
|
||||
SetBit(serializedHeader.Backwards,3,locUcaBook.Backwards[3]);
|
||||
if locHasProps then begin
|
||||
serializedHeader.BMP_Table1Length := Length(ucaFirstTable);
|
||||
serializedHeader.BMP_Table2Length := Length(TucaBmpSecondTableItem) *
|
||||
(Length(ucaSecondTable) * SizeOf(UInt24));
|
||||
serializedHeader.OBMP_Table1Length := Length(ucaoFirstTable) * SizeOf(Word);
|
||||
serializedHeader.OBMP_Table2Length := Length(TucaOBmpSecondTableItem) *
|
||||
(Length(ucaoSecondTable) * SizeOf(UInt24));
|
||||
serializedHeader.PropCount := locUcaProps^.ItemSize;
|
||||
serializedHeader.VariableLowLimit := locUcaProps^.VariableLowLimit;
|
||||
serializedHeader.VariableHighLimit := locUcaProps^.VariableHighLimit;
|
||||
end else begin
|
||||
serializedHeader.VariableLowLimit := High(Word);
|
||||
serializedHeader.VariableHighLimit := 0;
|
||||
end;
|
||||
serializedHeader.ChangedFields := 0;
|
||||
for e := Low(TCollationField) to High(TCollationField) do begin
|
||||
if (e in locItem.ChangedFields) then
|
||||
SetBit(serializedHeader.ChangedFields,Ord(e),True);
|
||||
end;
|
||||
ABinaryNativeEndianStream.Write(serializedHeader,SizeOf(serializedHeader));
|
||||
ReverseRecordBytes(serializedHeader);
|
||||
ABinaryOtherEndianStream.Write(serializedHeader,SizeOf(serializedHeader));
|
||||
if locHasProps then begin
|
||||
GenerateBinaryUCA_BmpTables(ABinaryNativeEndianStream,ABinaryOtherEndianStream,ucaFirstTable,ucaSecondTable);
|
||||
GenerateBinaryUCA_OBmpTables(ABinaryNativeEndianStream,ABinaryOtherEndianStream,ucaoFirstTable,ucaoSecondTable);
|
||||
GenerateBinaryUCA_PropTable(ABinaryNativeEndianStream,ABinaryOtherEndianStream,locUcaProps);
|
||||
end;
|
||||
finally
|
||||
locSequence.Clear();
|
||||
FreeUcaBook(locUcaProps);
|
||||
|
||||
@ -22,9 +22,10 @@
|
||||
program cldrparser;
|
||||
|
||||
{$mode objfpc}{$H+}
|
||||
{ $define WINCE_TEST}
|
||||
|
||||
uses
|
||||
SysUtils, classes, getopts,
|
||||
SysUtils, classes, getopts,{$ifdef WINCE}StreamIO,{$endif}
|
||||
cldrhelper, helper, cldrtest, cldrxml, unicodeset;
|
||||
|
||||
const
|
||||
@ -66,6 +67,12 @@ var
|
||||
idx, k : Integer;
|
||||
s : string;
|
||||
begin
|
||||
{$ifdef WINCE_TEST}
|
||||
ADataDir := ExtractFilePath(ParamStr(0))+'data';
|
||||
AOuputDir := ADataDir;
|
||||
ACollationFileName := 'sv.xml';
|
||||
exit(True);
|
||||
{$endif WINCE_TEST}
|
||||
if (ParamCount() = 0) then
|
||||
exit(False);
|
||||
Result := True;
|
||||
@ -101,14 +108,41 @@ end;
|
||||
var
|
||||
orderedChars : TOrderedCharacters;
|
||||
ucaBook : TUCA_DataBook;
|
||||
stream, streamNE, streamOE : TMemoryStream;
|
||||
stream, streamNE, streamOE, binaryStreamNE, binaryStreamOE : TMemoryStream;
|
||||
s, collationFileName, collationTypeName : string;
|
||||
i , c: Integer;
|
||||
collation : TCldrCollation;
|
||||
dataPath, outputPath : string;
|
||||
collationItem : TCldrCollationItem;
|
||||
testSuiteFlag : Boolean;
|
||||
{$ifdef WINCE}
|
||||
fs : TFileStream;
|
||||
{$endif WINCE}
|
||||
begin
|
||||
{$ifdef WINCE}
|
||||
s := ExtractFilePath(ParamStr(0))+'cldr-log.txt';
|
||||
DeleteFile(s);
|
||||
fs := TFileStream.Create(s,fmCreate);
|
||||
AssignStream(Output,fs);
|
||||
Rewrite(Output);
|
||||
s := ExtractFilePath(ParamStr(0))+'cldr-err.txt';
|
||||
DeleteFile(s);
|
||||
fs := TFileStream.Create(s,fmCreate);
|
||||
AssignStream(ErrOutput,fs);
|
||||
Rewrite(ErrOutput);
|
||||
{$endif WINCE}
|
||||
{$ifdef WINCE_TEST}
|
||||
testSuiteFlag := True;
|
||||
try
|
||||
exec_tests();
|
||||
except
|
||||
on e : Exception do begin
|
||||
WriteLn('Exception : '+e.Message);
|
||||
raise;
|
||||
end;
|
||||
end;
|
||||
exit;
|
||||
{$endif WINCE_TEST}
|
||||
dataPath := '';
|
||||
outputPath := '';
|
||||
collationFileName := '';
|
||||
@ -132,10 +166,12 @@ begin
|
||||
outputPath := dataPath
|
||||
else
|
||||
outputPath := IncludeTrailingPathDelimiter(outputPath);
|
||||
{$ifndef WINCE_TEST}
|
||||
if (ParamCount() = 0) then begin
|
||||
WriteLn(SUsageText);
|
||||
Halt(1);
|
||||
end;
|
||||
{$endif WINCE_TEST}
|
||||
if not(
|
||||
FileExists(dataPath+'UCA_Rules_SHORT.xml') and
|
||||
FileExists(dataPath+'allkeys.txt')
|
||||
@ -155,6 +191,8 @@ begin
|
||||
stream := nil;
|
||||
streamNE := nil;
|
||||
streamOE := nil;
|
||||
binaryStreamNE := nil;
|
||||
binaryStreamOE := nil;
|
||||
collation := TCldrCollation.Create();
|
||||
try
|
||||
ParseCollationDocument(collationFileName,collation,TCldrParserMode.HeaderParsing);
|
||||
@ -194,9 +232,12 @@ begin
|
||||
stream.Clear();
|
||||
streamNE := TMemoryStream.Create();
|
||||
streamOE := TMemoryStream.Create();
|
||||
binaryStreamNE := TMemoryStream.Create();
|
||||
binaryStreamOE := TMemoryStream.Create();
|
||||
s := COLLATION_FILE_PREFIX + ChangeFileExt(LowerCase(ExtractFileName(collationFileName)),'.pas');
|
||||
GenerateCdlrCollation(
|
||||
collation,collationTypeName,s,stream,streamNE,streamOE,
|
||||
binaryStreamNE,binaryStreamOE,
|
||||
orderedChars,ucaBook.Lines
|
||||
);
|
||||
stream.SaveToFile(ExtractFilePath(collationFileName)+s);
|
||||
@ -204,8 +245,20 @@ begin
|
||||
streamNE.SaveToFile(ExtractFilePath(collationFileName)+GenerateEndianIncludeFileName(s,ENDIAN_NATIVE));
|
||||
streamOE.SaveToFile(ExtractFilePath(collationFileName)+GenerateEndianIncludeFileName(s,ENDIAN_NON_NATIVE));
|
||||
end;
|
||||
if (binaryStreamNE.Size > 0) then begin
|
||||
binaryStreamNE.SaveToFile(
|
||||
ExtractFilePath(collationFileName) +
|
||||
ChangeFileExt(s,Format('_%s.bco',[ENDIAN_SUFFIX[ENDIAN_NATIVE]]))
|
||||
);
|
||||
binaryStreamOE.SaveToFile(
|
||||
ExtractFilePath(collationFileName) +
|
||||
ChangeFileExt(s,Format('_%s.bco',[ENDIAN_SUFFIX[ENDIAN_NON_NATIVE]]))
|
||||
);
|
||||
end;
|
||||
end;
|
||||
finally
|
||||
binaryStreamOE.Free();
|
||||
binaryStreamNE.Free();
|
||||
streamOE.Free();
|
||||
streamNE.Free();
|
||||
stream.Free();
|
||||
|
||||
@ -487,11 +487,23 @@ const
|
||||
var AFirstTable : TucaBmpFirstTable;
|
||||
var ASecondTable : TucaBmpSecondTable
|
||||
);
|
||||
procedure GenerateBinaryUCA_BmpTables(
|
||||
ANativeEndianStream,
|
||||
ANonNativeEndianStream : TStream;
|
||||
var AFirstTable : TucaBmpFirstTable;
|
||||
var ASecondTable : TucaBmpSecondTable
|
||||
);
|
||||
procedure GenerateUCA_PropTable(
|
||||
ADest : TStream;
|
||||
const APropBook : PUCA_PropBook;
|
||||
const AEndian : TEndianKind
|
||||
);
|
||||
procedure GenerateBinaryUCA_PropTable(
|
||||
// WARNING : files must be generated for each endianess (Little / Big)
|
||||
ANativeEndianStream,
|
||||
ANonNativeEndianStream : TStream;
|
||||
const APropBook : PUCA_PropBook
|
||||
);
|
||||
procedure GenerateUCA_OBmpTables(
|
||||
AStream,
|
||||
ANativeEndianStream,
|
||||
@ -499,6 +511,12 @@ const
|
||||
var AFirstTable : TucaOBmpFirstTable;
|
||||
var ASecondTable : TucaOBmpSecondTable
|
||||
);
|
||||
procedure GenerateBinaryUCA_OBmpTables(
|
||||
ANativeEndianStream,
|
||||
ANonNativeEndianStream : TStream;
|
||||
var AFirstTable : TucaOBmpFirstTable;
|
||||
var ASecondTable : TucaOBmpSecondTable
|
||||
);
|
||||
|
||||
procedure Parse_UnicodeData(
|
||||
ADataAStream : TMemoryStream;
|
||||
@ -611,7 +629,6 @@ const
|
||||
): PPropRec; inline;overload;
|
||||
procedure FromUCS4(const AValue : TUnicodeCodePoint; var AHighS, ALowS : Word);inline;
|
||||
function ToUCS4(const AHighS, ALowS : Word) : TUnicodeCodePoint; inline;
|
||||
//--------------------
|
||||
|
||||
type
|
||||
TBitOrder = 0..7;
|
||||
@ -640,6 +657,29 @@ type
|
||||
const ADataLen : Integer
|
||||
);
|
||||
|
||||
type
|
||||
TCollationName = string[128];
|
||||
TSerializedCollationHeader = packed record
|
||||
Base : TCollationName;
|
||||
Version : TCollationName;
|
||||
CollationName : TCollationName;
|
||||
VariableWeight : Byte;
|
||||
Backwards : Byte;
|
||||
BMP_Table1Length : DWord;
|
||||
BMP_Table2Length : DWord;
|
||||
OBMP_Table1Length : DWord;
|
||||
OBMP_Table2Length : DWord;
|
||||
PropCount : DWord;
|
||||
VariableLowLimit : Word;
|
||||
VariableHighLimit : Word;
|
||||
ChangedFields : Byte;
|
||||
end;
|
||||
PSerializedCollationHeader = ^TSerializedCollationHeader;
|
||||
|
||||
procedure ReverseRecordBytes(var AItem : TSerializedCollationHeader);
|
||||
procedure ReverseBytes(var AData; const ALength : Integer);
|
||||
procedure ReverseArray(var AValue; const AArrayLength, AItemSize : PtrInt);
|
||||
|
||||
resourcestring
|
||||
SInsufficientMemoryBuffer = 'Insufficient Memory Buffer';
|
||||
|
||||
@ -3294,6 +3334,28 @@ begin
|
||||
AddLine(ANonNativeEndianStream,' );' + sLineBreak);
|
||||
end;
|
||||
|
||||
procedure GenerateBinaryUCA_BmpTables(
|
||||
ANativeEndianStream,
|
||||
ANonNativeEndianStream : TStream;
|
||||
var AFirstTable : TucaBmpFirstTable;
|
||||
var ASecondTable : TucaBmpSecondTable
|
||||
);
|
||||
var
|
||||
i, j : Integer;
|
||||
value : UInt24;
|
||||
begin
|
||||
ANativeEndianStream.Write(AFirstTable[0],Length(AFirstTable));
|
||||
ANonNativeEndianStream.Write(AFirstTable[0],Length(AFirstTable));
|
||||
for i := Low(ASecondTable) to High(ASecondTable) do begin
|
||||
for j := Low(TucaBmpSecondTableItem) to High(TucaBmpSecondTableItem) do begin
|
||||
value := ASecondTable[i][j];
|
||||
ANativeEndianStream.Write(value,SizeOf(value));
|
||||
ReverseBytes(value,SizeOf(value));
|
||||
ANonNativeEndianStream.Write(value,SizeOf(value));
|
||||
end;
|
||||
end;
|
||||
end;
|
||||
|
||||
procedure GenerateUCA_PropTable(
|
||||
// WARNING : files must be generated for each endianess (Little / Big)
|
||||
ADest : TStream;
|
||||
@ -3336,6 +3398,17 @@ begin
|
||||
AddLine(' );' + sLineBreak);
|
||||
end;
|
||||
|
||||
procedure GenerateBinaryUCA_PropTable(
|
||||
// WARNING : files must be generated for each endianess (Little / Big)
|
||||
ANativeEndianStream,
|
||||
ANonNativeEndianStream : TStream;
|
||||
const APropBook : PUCA_PropBook
|
||||
);
|
||||
begin
|
||||
ANativeEndianStream.Write(APropBook^.Items^,APropBook^.ItemSize);
|
||||
ANonNativeEndianStream.Write(APropBook^.ItemsOtherEndian^,APropBook^.ItemSize);
|
||||
end;
|
||||
|
||||
procedure GenerateUCA_OBmpTables(
|
||||
AStream,
|
||||
ANativeEndianStream,
|
||||
@ -3410,7 +3483,34 @@ begin
|
||||
AddLine(ANonNativeEndianStream,' );' + sLineBreak);
|
||||
end;
|
||||
|
||||
//-------------------------------------------
|
||||
procedure GenerateBinaryUCA_OBmpTables(
|
||||
ANativeEndianStream,
|
||||
ANonNativeEndianStream : TStream;
|
||||
var AFirstTable : TucaOBmpFirstTable;
|
||||
var ASecondTable : TucaOBmpSecondTable
|
||||
);
|
||||
var
|
||||
i, j : Integer;
|
||||
locLine : string;
|
||||
wordValue : Word;
|
||||
value : UInt24;
|
||||
begin
|
||||
for i := Low(AFirstTable) to High(AFirstTable) do begin
|
||||
wordValue := AFirstTable[i];
|
||||
ANativeEndianStream.Write(wordValue,SizeOf(wordValue));
|
||||
ReverseBytes(wordValue,SizeOf(wordValue));
|
||||
ANonNativeEndianStream.Write(wordValue,SizeOf(wordValue));
|
||||
end;
|
||||
|
||||
for i := Low(ASecondTable) to High(ASecondTable) do begin
|
||||
for j := Low(TucaOBmpSecondTableItem) to High(TucaOBmpSecondTableItem) do begin
|
||||
value := ASecondTable[i][j];
|
||||
ANativeEndianStream.Write(value,SizeOf(value));
|
||||
ReverseBytes(value,SizeOf(value));
|
||||
ANonNativeEndianStream.Write(value,SizeOf(value));
|
||||
end;
|
||||
end;
|
||||
end;
|
||||
|
||||
type
|
||||
POBmpSecondTableItem = ^TOBmpSecondTableItem;
|
||||
@ -4103,6 +4203,17 @@ begin
|
||||
Result := r;
|
||||
end;
|
||||
|
||||
procedure ReverseRecordBytes(var AItem : TSerializedCollationHeader);
|
||||
begin
|
||||
ReverseBytes(AItem.BMP_Table1Length,SizeOf(AItem.BMP_Table1Length));
|
||||
ReverseBytes(AItem.BMP_Table2Length,SizeOf(AItem.BMP_Table2Length));
|
||||
ReverseBytes(AItem.OBMP_Table1Length,SizeOf(AItem.OBMP_Table1Length));
|
||||
ReverseBytes(AItem.OBMP_Table2Length,SizeOf(AItem.OBMP_Table2Length));
|
||||
ReverseBytes(AItem.PropCount,SizeOf(AItem.PropCount));
|
||||
ReverseBytes(AItem.VariableLowLimit,SizeOf(AItem.VariableLowLimit));
|
||||
ReverseBytes(AItem.VariableHighLimit,SizeOf(AItem.VariableHighLimit));
|
||||
end;
|
||||
|
||||
procedure ReverseBytes(var AData; const ALength : Integer);
|
||||
var
|
||||
i,j : PtrInt;
|
||||
|
||||
@ -31,7 +31,7 @@
|
||||
program unihelper;
|
||||
|
||||
{$mode objfpc}{$H+}
|
||||
{$typedadress on}
|
||||
{$typedaddress on}
|
||||
|
||||
uses
|
||||
SysUtils, Classes,
|
||||
@ -66,6 +66,7 @@ end;
|
||||
var
|
||||
dataPath, outputPath : string;
|
||||
stream, binStreamNE, binStreamOE, tmpStream : TMemoryStream;
|
||||
binaryStreamNE, binaryStreamOE : TMemoryStream;
|
||||
hangulSyllables : TCodePointRecArray;
|
||||
ucaBook : TUCA_DataBook;
|
||||
ucaPropBook : PUCA_PropBook;
|
||||
@ -95,6 +96,7 @@ var
|
||||
ucaoFirstTable : TucaoBmpFirstTable;
|
||||
ucaoSecondTable : TucaOBmpSecondTable;
|
||||
WL : Integer;
|
||||
serializedHeader : TSerializedCollationHeader;
|
||||
begin
|
||||
WriteLn(SUsage+sLineBreak);
|
||||
if (ParamCount > 0) then
|
||||
@ -125,6 +127,8 @@ begin
|
||||
Halt(1);
|
||||
end;
|
||||
|
||||
binaryStreamNE := nil;
|
||||
binaryStreamOE := nil;
|
||||
binStreamOE := nil;
|
||||
binStreamNE := nil;
|
||||
tmpStream := nil;
|
||||
@ -206,6 +210,8 @@ begin
|
||||
{$IFDEF UCA_TEST}
|
||||
uca_CheckProp_2y(ucaBook,ucaPropBook,@ucaoFirstTable,@ucaoSecondTable);
|
||||
{$ENDIF UCA_TEST}
|
||||
binaryStreamNE := TMemoryStream.Create();
|
||||
binaryStreamOE := TMemoryStream.Create();
|
||||
WriteLn('Generate UCA Props tables ...');
|
||||
binStreamNE.Clear();
|
||||
binStreamOE.Clear();
|
||||
@ -226,6 +232,37 @@ begin
|
||||
binStreamOE.SaveToFile(GenerateEndianIncludeFileName(s,ENDIAN_NON_NATIVE));
|
||||
binStreamNE.Clear();
|
||||
binStreamOE.Clear();
|
||||
// Binary DUCET
|
||||
FillChar(serializedHeader,SizeOf(TSerializedCollationHeader),0);
|
||||
serializedHeader.Version := ucaBook.Version;
|
||||
serializedHeader.CollationName := 'DUCET';//'Default Unicode Collation Element Table (DUCET)';
|
||||
serializedHeader.VariableWeight := Ord(ucaBook.VariableWeight);
|
||||
SetBit(serializedHeader.Backwards,0,ucaBook.Backwards[0]);
|
||||
SetBit(serializedHeader.Backwards,1,ucaBook.Backwards[1]);
|
||||
SetBit(serializedHeader.Backwards,2,ucaBook.Backwards[2]);
|
||||
SetBit(serializedHeader.Backwards,3,ucaBook.Backwards[3]);
|
||||
serializedHeader.BMP_Table1Length := Length(ucaFirstTable);
|
||||
serializedHeader.BMP_Table2Length := Length(TucaBmpSecondTableItem) *
|
||||
(Length(ucaSecondTable) * SizeOf(UInt24));
|
||||
serializedHeader.OBMP_Table1Length := Length(ucaoFirstTable) * SizeOf(Word);
|
||||
serializedHeader.OBMP_Table2Length := Length(TucaOBmpSecondTableItem) *
|
||||
(Length(ucaoSecondTable) * SizeOf(UInt24));
|
||||
serializedHeader.PropCount := ucaPropBook^.ItemSize;
|
||||
serializedHeader.VariableLowLimit := ucaPropBook^.VariableLowLimit;
|
||||
serializedHeader.VariableHighLimit := ucaPropBook^.VariableHighLimit;
|
||||
binaryStreamNE.Write(serializedHeader,SizeOf(serializedHeader));
|
||||
ReverseRecordBytes(serializedHeader);
|
||||
binaryStreamOE.Write(serializedHeader,SizeOf(serializedHeader));
|
||||
GenerateBinaryUCA_BmpTables(binaryStreamNE,binaryStreamOE,ucaFirstTable,ucaSecondTable);
|
||||
GenerateBinaryUCA_OBmpTables(binaryStreamNE,binaryStreamOE,ucaoFirstTable,ucaoSecondTable);
|
||||
GenerateBinaryUCA_PropTable(binaryStreamNE,binaryStreamOE,ucaPropBook);
|
||||
binaryStreamNE.SaveToFile(
|
||||
outputPath + Format('collation_ducet_%s.bco',[ENDIAN_SUFFIX[ENDIAN_NATIVE]])
|
||||
);
|
||||
binaryStreamOE.SaveToFile(
|
||||
outputPath + Format('collation_ducet_%s.bco',[ENDIAN_SUFFIX[ENDIAN_NON_NATIVE]])
|
||||
);
|
||||
// Binary DUCET - END
|
||||
|
||||
|
||||
stream.Clear();
|
||||
@ -386,6 +423,8 @@ begin
|
||||
end;
|
||||
stream.SaveToFile(outputPath + 'diff2.txt');
|
||||
finally
|
||||
binaryStreamOE.Free();
|
||||
binaryStreamNE.Free();
|
||||
tmpStream.Free();
|
||||
binStreamOE.Free();
|
||||
binStreamNE.Free();
|
||||
|
||||
Loading…
Reference in New Issue
Block a user