diff --git a/rtl/objpas/unicodedata.pas b/rtl/objpas/unicodedata.pas
index 2403a9b658..b00ee4877c 100644
--- a/rtl/objpas/unicodedata.pas
+++ b/rtl/objpas/unicodedata.pas
@@ -274,6 +274,7 @@ type
Props : PUCA_PropItemRec;
VariableLowLimit : Word;
VariableHighLimit : Word;
+ Dynamic : Boolean;
public
function IsVariable(const AWeight : PUCA_PropWeights) : Boolean; inline;
end;
@@ -320,8 +321,13 @@ type
function CompareSortKey(const A, B : TUCASortKey) : Integer;overload;
function CompareSortKey(const A : TUCASortKey; const B : array of Word) : Integer;overload;
- function RegisterCollation(const ACollation : PUCA_DataBook) : Boolean;
+ function RegisterCollation(const ACollation : PUCA_DataBook) : Boolean;overload;
+ function RegisterCollation(
+ const ADirectory,
+ ALanguage : string
+ ) : Boolean;overload;
function UnregisterCollation(const AName : ansistring): Boolean;
+ procedure UnregisterCollations(const AFreeDynamicCollations : Boolean);
function FindCollation(const AName : ansistring): PUCA_DataBook;overload;
function FindCollation(const AIndex : Integer): PUCA_DataBook;overload;
function GetCollationCount() : Integer;
@@ -330,6 +336,29 @@ type
const ABaseName : ansistring;
const AChangedFields : TCollationFields
);
+ function LoadCollation(
+ const AData : Pointer;
+ const ADataLength : Integer
+ ) : PUCA_DataBook;overload;
+ function LoadCollation(const AFileName : string) : PUCA_DataBook;overload;
+ function LoadCollation(
+ const ADirectory,
+ ALanguage : string
+ ) : PUCA_DataBook;overload;
+ procedure FreeCollation(AItem : PUCA_DataBook);
+
+type
+ TEndianKind = (Little, Big);
+const
+ ENDIAN_SUFFIX : array[TEndianKind] of string[2] = ('le','be');
+{$IFDEF ENDIAN_LITTLE}
+ ENDIAN_NATIVE = TEndianKind.Little;
+ ENDIAN_NON_NATIVE = TEndianKind.Big;
+{$ENDIF ENDIAN_LITTLE}
+{$IFDEF ENDIAN_BIG}
+ ENDIAN_NATIVE = TEndianKind.Big;
+ ENDIAN_NON_NATIVE = TEndianKind.Little;
+{$ENDIF ENDIAN_BIG}
resourcestring
SCollationNotFound = 'Collation not found : "%s".';
@@ -535,6 +564,21 @@ begin
Result := a <= Cardinal(b);
end;
+type
+ TBitOrder = 0..7;
+function IsBitON(const AData : Byte; const ABit : TBitOrder) : Boolean ;inline;
+begin
+ Result := ( ( AData and ( 1 shl ABit ) ) <> 0 );
+end;
+
+procedure SetBit(var AData : Byte; const ABit : TBitOrder; const AValue : Boolean);inline;
+begin
+ if AValue then
+ AData := AData or (1 shl (ABit mod 8))
+ else
+ AData := AData and ( not ( 1 shl ( ABit mod 8 ) ) );
+end;
+
var
CollationTable : array of PUCA_DataBook;
function IndexOfCollation(const AName : string) : Integer;
@@ -565,6 +609,23 @@ begin
end;
end;
+function RegisterCollation(const ADirectory, ALanguage : string) : Boolean;
+var
+ cl : PUCA_DataBook;
+begin
+ cl := LoadCollation(ADirectory,ALanguage);
+ if (cl = nil) then
+ exit(False);
+ try
+ Result := RegisterCollation(cl);
+ except
+ FreeCollation(cl);
+ raise;
+ end;
+ if not Result then
+ FreeCollation(cl);
+end;
+
function UnregisterCollation(const AName : ansistring): Boolean;
var
i, c : Integer;
@@ -582,6 +643,21 @@ begin
end;
end;
+procedure UnregisterCollations(const AFreeDynamicCollations : Boolean);
+var
+ i : Integer;
+ cl : PUCA_DataBook;
+begin
+ for i := Low(CollationTable) to High(CollationTable) do begin
+ if CollationTable[i].Dynamic then begin
+ cl := CollationTable[i];
+ CollationTable[i] := nil;
+ FreeCollation(cl);
+ end;
+ end;
+ SetLength(CollationTable,0);
+end;
+
function FindCollation(const AName : ansistring): PUCA_DataBook;overload;
var
i : Integer;
@@ -632,6 +708,190 @@ begin
p^.VariableLowLimit := base^.VariableHighLimit;
end;
+type
+ TSerializedCollationHeader = packed record
+ Base : TCollationName;
+ Version : TCollationName;
+ CollationName : TCollationName;
+ VariableWeight : Byte;
+ Backwards : Byte;
+ BMP_Table1Length : DWord;
+ BMP_Table2Length : DWord;
+ OBMP_Table1Length : DWord;
+ OBMP_Table2Length : DWord;
+ PropCount : DWord;
+ VariableLowLimit : Word;
+ VariableHighLimit : Word;
+ ChangedFields : Byte;
+ end;
+ PSerializedCollationHeader = ^TSerializedCollationHeader;
+
+procedure FreeCollation(AItem : PUCA_DataBook);
+var
+ h : PSerializedCollationHeader;
+begin
+ if (AItem = nil) or not(AItem^.Dynamic) then
+ exit;
+ h := PSerializedCollationHeader(PtrUInt(AItem) + SizeOf(TUCA_DataBook));
+ if (AItem^.BMP_Table1 <> nil) then
+ FreeMem(AItem^.BMP_Table1,h^.BMP_Table1Length);
+ if (AItem^.BMP_Table2 <> nil) then
+ FreeMem(AItem^.BMP_Table2,h^.BMP_Table2Length);
+ if (AItem^.OBMP_Table1 <> nil) then
+ FreeMem(AItem^.OBMP_Table1,h^.OBMP_Table1Length);
+ if (AItem^.OBMP_Table2 <> nil) then
+ FreeMem(AItem^.OBMP_Table2,h^.OBMP_Table2Length);
+ if (AItem^.Props <> nil) then
+ FreeMem(AItem^.Props,h^.PropCount);
+ FreeMem(AItem,(SizeOf(TUCA_DataBook)+SizeOf(TSerializedCollationHeader)));
+end;
+
+function LoadCollation(
+ const AData : Pointer;
+ const ADataLength : Integer
+) : PUCA_DataBook;
+var
+ dataPointer : PByte;
+ readedLength : LongInt;
+
+ function ReadBuffer(ADest : Pointer; ALength : LongInt) : Boolean;
+ begin
+ Result := (readedLength + ALength) <= ADataLength;
+ if not result then
+ exit;
+ Move(dataPointer^,ADest^,ALength);
+ Inc(dataPointer,ALength);
+ readedLength := readedLength + ALength;
+ end;
+
+var
+ r : PUCA_DataBook;
+ h : PSerializedCollationHeader;
+ cfs : TCollationFields;
+ i : Integer;
+ baseName : TCollationName;
+begin
+ readedLength := 0;
+ dataPointer := AData;
+ r := AllocMem((SizeOf(TUCA_DataBook)+SizeOf(TSerializedCollationHeader)));
+ try
+ h := PSerializedCollationHeader(PtrUInt(r) + SizeOf(TUCA_DataBook));
+ if not ReadBuffer(h,SizeOf(TSerializedCollationHeader)) then
+ exit;
+ r^.Version := h^.Version;
+ r^.CollationName := h^.CollationName;
+ r^.VariableWeight := TUCA_VariableKind(h^.VariableWeight);
+ r^.Backwards[0] := IsBitON(h^.Backwards,0);
+ r^.Backwards[1] := IsBitON(h^.Backwards,1);
+ r^.Backwards[2] := IsBitON(h^.Backwards,2);
+ r^.Backwards[3] := IsBitON(h^.Backwards,3);
+ if (h^.BMP_Table1Length > 0) then begin
+ r^.BMP_Table1 := GetMem(h^.BMP_Table1Length);
+ if not ReadBuffer(r^.BMP_Table1,h^.BMP_Table1Length) then
+ exit;
+ end;
+ if (h^.BMP_Table2Length > 0) then begin
+ r^.BMP_Table2 := GetMem(h^.BMP_Table2Length);
+ if not ReadBuffer(r^.BMP_Table2,h^.BMP_Table2Length) then
+ exit;
+ end;
+ if (h^.OBMP_Table1Length > 0) then begin
+ r^.OBMP_Table1 := GetMem(h^.OBMP_Table1Length);
+ if not ReadBuffer(r^.OBMP_Table1,h^.OBMP_Table1Length) then
+ exit;
+ end;
+ if (h^.OBMP_Table2Length > 0) then begin
+ r^.OBMP_Table2 := GetMem(h^.OBMP_Table2Length);
+ if not ReadBuffer(r^.OBMP_Table2,h^.OBMP_Table2Length) then
+ exit;
+ end;
+ r^.PropCount := h^.PropCount;
+ if (h^.PropCount > 0) then begin
+ r^.Props := GetMem(h^.PropCount);
+ if not ReadBuffer(r^.Props,h^.PropCount) then
+ exit;
+ end;
+ r^.VariableLowLimit := h^.VariableLowLimit;
+ r^.VariableHighLimit := h^.VariableHighLimit;
+
+ cfs := [];
+ for i := Ord(Low(TCollationField)) to Ord(High(TCollationField)) do begin
+ if IsBitON(h^.ChangedFields,i) then
+ cfs := cfs + [TCollationField(i)];
+ end;
+ if (h^.Base <> '') then
+ baseName := h^.Base
+ else if (h^.CollationName <> ROOT_COLLATION_NAME) then
+ baseName := ROOT_COLLATION_NAME
+ else
+ baseName := '';
+ if (baseName <> '') then
+ PrepareCollation(r,baseName,cfs);
+ r^.Dynamic := True;
+ Result := r;
+ except
+ FreeCollation(r);
+ raise;
+ end;
+end;
+
+{$PUSH}
+function LoadCollation(const AFileName : string) : PUCA_DataBook;
+const
+ BLOCK_SIZE = 16*1024;
+var
+ f : File of Byte;
+ locSize, locReaded, c : LongInt;
+ locBuffer : PByte;
+ locBlockSize : LongInt;
+begin
+ Result := nil;
+{$I-}
+ if (AFileName = '') then
+ exit;
+ Assign(f,AFileName);
+ Reset(f);
+ try
+ if (IOResult <> 0) then
+ exit;
+ locSize := FileSize(f);
+ if (locSize < SizeOf(TSerializedCollationHeader)) then
+ exit;
+ locBuffer := GetMem(locSize);
+ try
+ locBlockSize := BLOCK_SIZE;
+ locReaded := 0;
+ while (locReaded < locSize) do begin
+ if (locBlockSize > (locSize-locReaded)) then
+ locBlockSize := locSize-locReaded;
+ BlockRead(f,locBuffer[locReaded],locBlockSize,c);
+ if (IOResult <> 0) or (c <= 0) then
+ exit;
+ locReaded := locReaded + c;
+ end;
+ Result := LoadCollation(locBuffer,locSize);
+ finally
+ FreeMem(locBuffer,locSize);
+ end;
+ finally
+ Close(f);
+ end;
+end;
+{$POP}
+
+function LoadCollation(const ADirectory, ALanguage : string) : PUCA_DataBook;
+var
+ fileName : string;
+begin
+ fileName := ADirectory;
+ if (fileName <> '') then begin
+ if (fileName[Length(fileName)] <> DirectorySeparator) then
+ fileName := fileName + DirectorySeparator;
+ end;
+ fileName := fileName + 'collation_' + ALanguage + '_' + ENDIAN_SUFFIX[ENDIAN_NATIVE] + '.bco';
+ Result := LoadCollation(fileName);
+end;
+
{$INCLUDE unicodedata.inc}
{$IFDEF ENDIAN_LITTLE}
{$INCLUDE unicodedata_le.inc}
@@ -1026,21 +1286,6 @@ begin
end;
end;
-type
- TBitOrder = 0..7;
-function IsBitON(const AData : Byte; const ABit : TBitOrder) : Boolean ;inline;
-begin
- Result := ( ( AData and ( 1 shl ABit ) ) <> 0 );
-end;
-
-procedure SetBit(var AData : Byte; const ABit : TBitOrder; const AValue : Boolean);inline;
-begin
- if AValue then
- AData := AData or (1 shl (ABit mod 8))
- else
- AData := AData and ( not ( 1 shl ( ABit mod 8 ) ) );
-end;
-
{ TUCA_PropItemContextTreeNodeRec }
function TUCA_PropItemContextTreeNodeRec.GetLeftNode: PUCA_PropItemContextTreeNodeRec;
diff --git a/utils/unicode/cldrhelper.pas b/utils/unicode/cldrhelper.pas
index e5324f6609..a1495244f5 100644
--- a/utils/unicode/cldrhelper.pas
+++ b/utils/unicode/cldrhelper.pas
@@ -208,14 +208,16 @@ type
) : Integer;
function FindCollationDefaultItemName(ACollation : TCldrCollation) : string;
procedure GenerateCdlrCollation(
- ACollation : TCldrCollation;
- AItemName : string;
- AStoreName : string;
+ ACollation : TCldrCollation;
+ AItemName : string;
+ AStoreName : string;
AStream,
ANativeEndianStream,
- AOtherEndianStream : TStream;
- ARootChars : TOrderedCharacters;
- ARootWeigths : TUCA_LineRecArray
+ AOtherEndianStream,
+ ABinaryNativeEndianStream,
+ ABinaryOtherEndianStream : TStream;
+ ARootChars : TOrderedCharacters;
+ ARootWeigths : TUCA_LineRecArray
);
procedure GenerateUCA_CLDR_Head(
@@ -1635,14 +1637,16 @@ begin
end;
procedure GenerateCdlrCollation(
- ACollation : TCldrCollation;
- AItemName : string;
- AStoreName : string;
+ ACollation : TCldrCollation;
+ AItemName : string;
+ AStoreName : string;
AStream,
ANativeEndianStream,
- AOtherEndianStream : TStream;
- ARootChars : TOrderedCharacters;
- ARootWeigths : TUCA_LineRecArray
+ AOtherEndianStream,
+ ABinaryNativeEndianStream,
+ ABinaryOtherEndianStream : TStream;
+ ARootChars : TOrderedCharacters;
+ ARootWeigths : TUCA_LineRecArray
);
procedure AddLine(const ALine : ansistring; ADestStream : TStream);
@@ -1665,6 +1669,8 @@ var
ucaoSecondTable : TucaOBmpSecondTable;
locHasProps : Boolean;
s : string;
+ serializedHeader : TSerializedCollationHeader;
+ e : TCollationField;
begin
locItem := ACollation.Find(AItemName);
if (locItem = nil) then
@@ -1707,6 +1713,43 @@ begin
AddLine('{$endif FPC_LITTLE_ENDIAN}',AStream);
end;
GenerateUCA_CLDR_Registration(AStream,@locUcaBook);
+
+ FillChar(serializedHeader,SizeOf(TSerializedCollationHeader),0);
+ serializedHeader.Base := locItem.Base;
+ serializedHeader.Version := ACollation.Version;
+ serializedHeader.CollationName := ACollation.Language;
+ serializedHeader.VariableWeight := Ord(locUcaBook.VariableWeight);
+ SetBit(serializedHeader.Backwards,0,locUcaBook.Backwards[0]);
+ SetBit(serializedHeader.Backwards,1,locUcaBook.Backwards[1]);
+ SetBit(serializedHeader.Backwards,2,locUcaBook.Backwards[2]);
+ SetBit(serializedHeader.Backwards,3,locUcaBook.Backwards[3]);
+ if locHasProps then begin
+ serializedHeader.BMP_Table1Length := Length(ucaFirstTable);
+ serializedHeader.BMP_Table2Length := Length(TucaBmpSecondTableItem) *
+ (Length(ucaSecondTable) * SizeOf(UInt24));
+ serializedHeader.OBMP_Table1Length := Length(ucaoFirstTable) * SizeOf(Word);
+ serializedHeader.OBMP_Table2Length := Length(TucaOBmpSecondTableItem) *
+ (Length(ucaoSecondTable) * SizeOf(UInt24));
+ serializedHeader.PropCount := locUcaProps^.ItemSize;
+ serializedHeader.VariableLowLimit := locUcaProps^.VariableLowLimit;
+ serializedHeader.VariableHighLimit := locUcaProps^.VariableHighLimit;
+ end else begin
+ serializedHeader.VariableLowLimit := High(Word);
+ serializedHeader.VariableHighLimit := 0;
+ end;
+ serializedHeader.ChangedFields := 0;
+ for e := Low(TCollationField) to High(TCollationField) do begin
+ if (e in locItem.ChangedFields) then
+ SetBit(serializedHeader.ChangedFields,Ord(e),True);
+ end;
+ ABinaryNativeEndianStream.Write(serializedHeader,SizeOf(serializedHeader));
+ ReverseRecordBytes(serializedHeader);
+ ABinaryOtherEndianStream.Write(serializedHeader,SizeOf(serializedHeader));
+ if locHasProps then begin
+ GenerateBinaryUCA_BmpTables(ABinaryNativeEndianStream,ABinaryOtherEndianStream,ucaFirstTable,ucaSecondTable);
+ GenerateBinaryUCA_OBmpTables(ABinaryNativeEndianStream,ABinaryOtherEndianStream,ucaoFirstTable,ucaoSecondTable);
+ GenerateBinaryUCA_PropTable(ABinaryNativeEndianStream,ABinaryOtherEndianStream,locUcaProps);
+ end;
finally
locSequence.Clear();
FreeUcaBook(locUcaProps);
diff --git a/utils/unicode/cldrparser.lpr b/utils/unicode/cldrparser.lpr
index 9fa60926f2..a8d98796c1 100644
--- a/utils/unicode/cldrparser.lpr
+++ b/utils/unicode/cldrparser.lpr
@@ -22,9 +22,10 @@
program cldrparser;
{$mode objfpc}{$H+}
+{ $define WINCE_TEST}
uses
- SysUtils, classes, getopts,
+ SysUtils, classes, getopts,{$ifdef WINCE}StreamIO,{$endif}
cldrhelper, helper, cldrtest, cldrxml, unicodeset;
const
@@ -66,6 +67,12 @@ var
idx, k : Integer;
s : string;
begin
+{$ifdef WINCE_TEST}
+ ADataDir := ExtractFilePath(ParamStr(0))+'data';
+ AOuputDir := ADataDir;
+ ACollationFileName := 'sv.xml';
+ exit(True);
+{$endif WINCE_TEST}
if (ParamCount() = 0) then
exit(False);
Result := True;
@@ -101,14 +108,41 @@ end;
var
orderedChars : TOrderedCharacters;
ucaBook : TUCA_DataBook;
- stream, streamNE, streamOE : TMemoryStream;
+ stream, streamNE, streamOE, binaryStreamNE, binaryStreamOE : TMemoryStream;
s, collationFileName, collationTypeName : string;
i , c: Integer;
collation : TCldrCollation;
dataPath, outputPath : string;
collationItem : TCldrCollationItem;
testSuiteFlag : Boolean;
+{$ifdef WINCE}
+ fs : TFileStream;
+{$endif WINCE}
begin
+{$ifdef WINCE}
+ s := ExtractFilePath(ParamStr(0))+'cldr-log.txt';
+ DeleteFile(s);
+ fs := TFileStream.Create(s,fmCreate);
+ AssignStream(Output,fs);
+ Rewrite(Output);
+ s := ExtractFilePath(ParamStr(0))+'cldr-err.txt';
+ DeleteFile(s);
+ fs := TFileStream.Create(s,fmCreate);
+ AssignStream(ErrOutput,fs);
+ Rewrite(ErrOutput);
+{$endif WINCE}
+{$ifdef WINCE_TEST}
+ testSuiteFlag := True;
+ try
+ exec_tests();
+ except
+ on e : Exception do begin
+ WriteLn('Exception : '+e.Message);
+ raise;
+ end;
+ end;
+ exit;
+{$endif WINCE_TEST}
dataPath := '';
outputPath := '';
collationFileName := '';
@@ -132,10 +166,12 @@ begin
outputPath := dataPath
else
outputPath := IncludeTrailingPathDelimiter(outputPath);
+{$ifndef WINCE_TEST}
if (ParamCount() = 0) then begin
WriteLn(SUsageText);
Halt(1);
end;
+{$endif WINCE_TEST}
if not(
FileExists(dataPath+'UCA_Rules_SHORT.xml') and
FileExists(dataPath+'allkeys.txt')
@@ -155,6 +191,8 @@ begin
stream := nil;
streamNE := nil;
streamOE := nil;
+ binaryStreamNE := nil;
+ binaryStreamOE := nil;
collation := TCldrCollation.Create();
try
ParseCollationDocument(collationFileName,collation,TCldrParserMode.HeaderParsing);
@@ -194,9 +232,12 @@ begin
stream.Clear();
streamNE := TMemoryStream.Create();
streamOE := TMemoryStream.Create();
+ binaryStreamNE := TMemoryStream.Create();
+ binaryStreamOE := TMemoryStream.Create();
s := COLLATION_FILE_PREFIX + ChangeFileExt(LowerCase(ExtractFileName(collationFileName)),'.pas');
GenerateCdlrCollation(
collation,collationTypeName,s,stream,streamNE,streamOE,
+ binaryStreamNE,binaryStreamOE,
orderedChars,ucaBook.Lines
);
stream.SaveToFile(ExtractFilePath(collationFileName)+s);
@@ -204,8 +245,20 @@ begin
streamNE.SaveToFile(ExtractFilePath(collationFileName)+GenerateEndianIncludeFileName(s,ENDIAN_NATIVE));
streamOE.SaveToFile(ExtractFilePath(collationFileName)+GenerateEndianIncludeFileName(s,ENDIAN_NON_NATIVE));
end;
+ if (binaryStreamNE.Size > 0) then begin
+ binaryStreamNE.SaveToFile(
+ ExtractFilePath(collationFileName) +
+ ChangeFileExt(s,Format('_%s.bco',[ENDIAN_SUFFIX[ENDIAN_NATIVE]]))
+ );
+ binaryStreamOE.SaveToFile(
+ ExtractFilePath(collationFileName) +
+ ChangeFileExt(s,Format('_%s.bco',[ENDIAN_SUFFIX[ENDIAN_NON_NATIVE]]))
+ );
+ end;
end;
finally
+ binaryStreamOE.Free();
+ binaryStreamNE.Free();
streamOE.Free();
streamNE.Free();
stream.Free();
diff --git a/utils/unicode/helper.pas b/utils/unicode/helper.pas
index e171a72672..1e10910f5d 100644
--- a/utils/unicode/helper.pas
+++ b/utils/unicode/helper.pas
@@ -487,11 +487,23 @@ const
var AFirstTable : TucaBmpFirstTable;
var ASecondTable : TucaBmpSecondTable
);
+ procedure GenerateBinaryUCA_BmpTables(
+ ANativeEndianStream,
+ ANonNativeEndianStream : TStream;
+ var AFirstTable : TucaBmpFirstTable;
+ var ASecondTable : TucaBmpSecondTable
+ );
procedure GenerateUCA_PropTable(
ADest : TStream;
const APropBook : PUCA_PropBook;
const AEndian : TEndianKind
);
+ procedure GenerateBinaryUCA_PropTable(
+ // WARNING : files must be generated for each endianess (Little / Big)
+ ANativeEndianStream,
+ ANonNativeEndianStream : TStream;
+ const APropBook : PUCA_PropBook
+ );
procedure GenerateUCA_OBmpTables(
AStream,
ANativeEndianStream,
@@ -499,6 +511,12 @@ const
var AFirstTable : TucaOBmpFirstTable;
var ASecondTable : TucaOBmpSecondTable
);
+ procedure GenerateBinaryUCA_OBmpTables(
+ ANativeEndianStream,
+ ANonNativeEndianStream : TStream;
+ var AFirstTable : TucaOBmpFirstTable;
+ var ASecondTable : TucaOBmpSecondTable
+ );
procedure Parse_UnicodeData(
ADataAStream : TMemoryStream;
@@ -611,7 +629,6 @@ const
): PPropRec; inline;overload;
procedure FromUCS4(const AValue : TUnicodeCodePoint; var AHighS, ALowS : Word);inline;
function ToUCS4(const AHighS, ALowS : Word) : TUnicodeCodePoint; inline;
-//--------------------
type
TBitOrder = 0..7;
@@ -640,6 +657,29 @@ type
const ADataLen : Integer
);
+type
+ TCollationName = string[128];
+ TSerializedCollationHeader = packed record
+ Base : TCollationName;
+ Version : TCollationName;
+ CollationName : TCollationName;
+ VariableWeight : Byte;
+ Backwards : Byte;
+ BMP_Table1Length : DWord;
+ BMP_Table2Length : DWord;
+ OBMP_Table1Length : DWord;
+ OBMP_Table2Length : DWord;
+ PropCount : DWord;
+ VariableLowLimit : Word;
+ VariableHighLimit : Word;
+ ChangedFields : Byte;
+ end;
+ PSerializedCollationHeader = ^TSerializedCollationHeader;
+
+ procedure ReverseRecordBytes(var AItem : TSerializedCollationHeader);
+ procedure ReverseBytes(var AData; const ALength : Integer);
+ procedure ReverseArray(var AValue; const AArrayLength, AItemSize : PtrInt);
+
resourcestring
SInsufficientMemoryBuffer = 'Insufficient Memory Buffer';
@@ -3294,6 +3334,28 @@ begin
AddLine(ANonNativeEndianStream,' );' + sLineBreak);
end;
+procedure GenerateBinaryUCA_BmpTables(
+ ANativeEndianStream,
+ ANonNativeEndianStream : TStream;
+ var AFirstTable : TucaBmpFirstTable;
+ var ASecondTable : TucaBmpSecondTable
+);
+var
+ i, j : Integer;
+ value : UInt24;
+begin
+ ANativeEndianStream.Write(AFirstTable[0],Length(AFirstTable));
+ ANonNativeEndianStream.Write(AFirstTable[0],Length(AFirstTable));
+ for i := Low(ASecondTable) to High(ASecondTable) do begin
+ for j := Low(TucaBmpSecondTableItem) to High(TucaBmpSecondTableItem) do begin
+ value := ASecondTable[i][j];
+ ANativeEndianStream.Write(value,SizeOf(value));
+ ReverseBytes(value,SizeOf(value));
+ ANonNativeEndianStream.Write(value,SizeOf(value));
+ end;
+ end;
+end;
+
procedure GenerateUCA_PropTable(
// WARNING : files must be generated for each endianess (Little / Big)
ADest : TStream;
@@ -3336,6 +3398,17 @@ begin
AddLine(' );' + sLineBreak);
end;
+procedure GenerateBinaryUCA_PropTable(
+// WARNING : files must be generated for each endianess (Little / Big)
+ ANativeEndianStream,
+ ANonNativeEndianStream : TStream;
+ const APropBook : PUCA_PropBook
+);
+begin
+ ANativeEndianStream.Write(APropBook^.Items^,APropBook^.ItemSize);
+ ANonNativeEndianStream.Write(APropBook^.ItemsOtherEndian^,APropBook^.ItemSize);
+end;
+
procedure GenerateUCA_OBmpTables(
AStream,
ANativeEndianStream,
@@ -3410,7 +3483,34 @@ begin
AddLine(ANonNativeEndianStream,' );' + sLineBreak);
end;
-//-------------------------------------------
+procedure GenerateBinaryUCA_OBmpTables(
+ ANativeEndianStream,
+ ANonNativeEndianStream : TStream;
+ var AFirstTable : TucaOBmpFirstTable;
+ var ASecondTable : TucaOBmpSecondTable
+);
+var
+ i, j : Integer;
+ locLine : string;
+ wordValue : Word;
+ value : UInt24;
+begin
+ for i := Low(AFirstTable) to High(AFirstTable) do begin
+ wordValue := AFirstTable[i];
+ ANativeEndianStream.Write(wordValue,SizeOf(wordValue));
+ ReverseBytes(wordValue,SizeOf(wordValue));
+ ANonNativeEndianStream.Write(wordValue,SizeOf(wordValue));
+ end;
+
+ for i := Low(ASecondTable) to High(ASecondTable) do begin
+ for j := Low(TucaOBmpSecondTableItem) to High(TucaOBmpSecondTableItem) do begin
+ value := ASecondTable[i][j];
+ ANativeEndianStream.Write(value,SizeOf(value));
+ ReverseBytes(value,SizeOf(value));
+ ANonNativeEndianStream.Write(value,SizeOf(value));
+ end;
+ end;
+end;
type
POBmpSecondTableItem = ^TOBmpSecondTableItem;
@@ -4103,6 +4203,17 @@ begin
Result := r;
end;
+procedure ReverseRecordBytes(var AItem : TSerializedCollationHeader);
+begin
+ ReverseBytes(AItem.BMP_Table1Length,SizeOf(AItem.BMP_Table1Length));
+ ReverseBytes(AItem.BMP_Table2Length,SizeOf(AItem.BMP_Table2Length));
+ ReverseBytes(AItem.OBMP_Table1Length,SizeOf(AItem.OBMP_Table1Length));
+ ReverseBytes(AItem.OBMP_Table2Length,SizeOf(AItem.OBMP_Table2Length));
+ ReverseBytes(AItem.PropCount,SizeOf(AItem.PropCount));
+ ReverseBytes(AItem.VariableLowLimit,SizeOf(AItem.VariableLowLimit));
+ ReverseBytes(AItem.VariableHighLimit,SizeOf(AItem.VariableHighLimit));
+end;
+
procedure ReverseBytes(var AData; const ALength : Integer);
var
i,j : PtrInt;
diff --git a/utils/unicode/unihelper.lpr b/utils/unicode/unihelper.lpr
index 794f901b5d..f1b3de9e2e 100644
--- a/utils/unicode/unihelper.lpr
+++ b/utils/unicode/unihelper.lpr
@@ -31,7 +31,7 @@
program unihelper;
{$mode objfpc}{$H+}
-{$typedadress on}
+{$typedaddress on}
uses
SysUtils, Classes,
@@ -66,6 +66,7 @@ end;
var
dataPath, outputPath : string;
stream, binStreamNE, binStreamOE, tmpStream : TMemoryStream;
+ binaryStreamNE, binaryStreamOE : TMemoryStream;
hangulSyllables : TCodePointRecArray;
ucaBook : TUCA_DataBook;
ucaPropBook : PUCA_PropBook;
@@ -95,6 +96,7 @@ var
ucaoFirstTable : TucaoBmpFirstTable;
ucaoSecondTable : TucaOBmpSecondTable;
WL : Integer;
+ serializedHeader : TSerializedCollationHeader;
begin
WriteLn(SUsage+sLineBreak);
if (ParamCount > 0) then
@@ -125,6 +127,8 @@ begin
Halt(1);
end;
+ binaryStreamNE := nil;
+ binaryStreamOE := nil;
binStreamOE := nil;
binStreamNE := nil;
tmpStream := nil;
@@ -206,6 +210,8 @@ begin
{$IFDEF UCA_TEST}
uca_CheckProp_2y(ucaBook,ucaPropBook,@ucaoFirstTable,@ucaoSecondTable);
{$ENDIF UCA_TEST}
+ binaryStreamNE := TMemoryStream.Create();
+ binaryStreamOE := TMemoryStream.Create();
WriteLn('Generate UCA Props tables ...');
binStreamNE.Clear();
binStreamOE.Clear();
@@ -226,6 +232,37 @@ begin
binStreamOE.SaveToFile(GenerateEndianIncludeFileName(s,ENDIAN_NON_NATIVE));
binStreamNE.Clear();
binStreamOE.Clear();
+// Binary DUCET
+ FillChar(serializedHeader,SizeOf(TSerializedCollationHeader),0);
+ serializedHeader.Version := ucaBook.Version;
+ serializedHeader.CollationName := 'DUCET';//'Default Unicode Collation Element Table (DUCET)';
+ serializedHeader.VariableWeight := Ord(ucaBook.VariableWeight);
+ SetBit(serializedHeader.Backwards,0,ucaBook.Backwards[0]);
+ SetBit(serializedHeader.Backwards,1,ucaBook.Backwards[1]);
+ SetBit(serializedHeader.Backwards,2,ucaBook.Backwards[2]);
+ SetBit(serializedHeader.Backwards,3,ucaBook.Backwards[3]);
+ serializedHeader.BMP_Table1Length := Length(ucaFirstTable);
+ serializedHeader.BMP_Table2Length := Length(TucaBmpSecondTableItem) *
+ (Length(ucaSecondTable) * SizeOf(UInt24));
+ serializedHeader.OBMP_Table1Length := Length(ucaoFirstTable) * SizeOf(Word);
+ serializedHeader.OBMP_Table2Length := Length(TucaOBmpSecondTableItem) *
+ (Length(ucaoSecondTable) * SizeOf(UInt24));
+ serializedHeader.PropCount := ucaPropBook^.ItemSize;
+ serializedHeader.VariableLowLimit := ucaPropBook^.VariableLowLimit;
+ serializedHeader.VariableHighLimit := ucaPropBook^.VariableHighLimit;
+ binaryStreamNE.Write(serializedHeader,SizeOf(serializedHeader));
+ ReverseRecordBytes(serializedHeader);
+ binaryStreamOE.Write(serializedHeader,SizeOf(serializedHeader));
+ GenerateBinaryUCA_BmpTables(binaryStreamNE,binaryStreamOE,ucaFirstTable,ucaSecondTable);
+ GenerateBinaryUCA_OBmpTables(binaryStreamNE,binaryStreamOE,ucaoFirstTable,ucaoSecondTable);
+ GenerateBinaryUCA_PropTable(binaryStreamNE,binaryStreamOE,ucaPropBook);
+ binaryStreamNE.SaveToFile(
+ outputPath + Format('collation_ducet_%s.bco',[ENDIAN_SUFFIX[ENDIAN_NATIVE]])
+ );
+ binaryStreamOE.SaveToFile(
+ outputPath + Format('collation_ducet_%s.bco',[ENDIAN_SUFFIX[ENDIAN_NON_NATIVE]])
+ );
+// Binary DUCET - END
stream.Clear();
@@ -386,6 +423,8 @@ begin
end;
stream.SaveToFile(outputPath + 'diff2.txt');
finally
+ binaryStreamOE.Free();
+ binaryStreamNE.Free();
tmpStream.Free();
binStreamOE.Free();
binStreamNE.Free();