From 9f023fcfe0d5559e2af1b555670c5119e285f711 Mon Sep 17 00:00:00 2001 From: paul Date: Fri, 16 Aug 2013 09:32:27 +0000 Subject: [PATCH] rtl, utils: apply patch of Inoussa which adds dynamic loading of charset binary files (issue #0024862) git-svn-id: trunk@25262 - --- rtl/inc/charset.pp | 166 ++++++++++++++++++++++++++++++++++++++++++++- utils/creumap.pp | 66 ++++++++++++++++++ 2 files changed, 230 insertions(+), 2 deletions(-) diff --git a/rtl/inc/charset.pp b/rtl/inc/charset.pp index d8d911009f..07efb7c1eb 100644 --- a/rtl/inc/charset.pp +++ b/rtl/inc/charset.pp @@ -15,6 +15,7 @@ **********************************************************************} {$mode objfpc} {$pointermath on} +{$PACKENUM 1} unit charset; interface @@ -31,14 +32,14 @@ unit charset; umf_unused); punicodecharmapping = ^tunicodecharmapping; - tunicodecharmapping = record + tunicodecharmapping = packed record unicode : tunicodechar; flag : tunicodecharmappingflag; reserved : byte; end; preversecharmapping = ^treversecharmapping; - treversecharmapping = record + treversecharmapping = packed record unicode : tunicodechar; char1 : Byte; char2 : Byte; @@ -59,8 +60,26 @@ unit charset; tcp2unicode = class(tcsconvert) end; + TSerializedMapHeader = packed record + cpName : string[20]; + cp : UInt16; + mapLength : UInt32; + lastChar : Int32; + reverseMapLength : UInt32; + end; + + const + BINARY_MAPPING_FILE_EXT = '.bcm'; + function loadunicodemapping(const cpname,f : string; cp :word) : punicodemap; + function loadbinaryunicodemapping(const directory,cpname : string) : punicodemap;overload; + function loadbinaryunicodemapping(const filename : string) : punicodemap;overload; + function loadbinaryunicodemapping( + const AData : Pointer; + const ADataLength : Integer + ) : punicodemap;overload; procedure registermapping(p : punicodemap); + function registerbinarymapping(const directory,cpname : string):Boolean; function getmap(const s : string) : punicodemap; function getmap(cp : word) : punicodemap; function mappingavailable(const s : string) : boolean;inline; @@ -371,6 +390,136 @@ unit charset; loadunicodemapping:=p; end; + + function loadbinaryunicodemapping(const directory, cpname : string) : punicodemap; + const + {$IFDEF ENDIAN_LITTLE} + ENDIAN_SUFFIX = 'le'; + {$ENDIF ENDIAN_LITTLE} + {$IFDEF ENDIAN_BIG} + ENDIAN_SUFFIX = 'be'; + {$ENDIF ENDIAN_BIG} + var + fileName : string; + begin + fileName := directory; + if (fileName <> '') then begin + if (fileName[Length(fileName)] <> DirectorySeparator) then + fileName := fileName + DirectorySeparator; + end; + fileName := fileName + cpname + '_' + ENDIAN_SUFFIX + BINARY_MAPPING_FILE_EXT; + Result := loadbinaryunicodemapping(fileName); + end; + + {$PUSH} + {$I-} + function loadbinaryunicodemapping(const filename : string) : punicodemap; + const + BLOCK_SIZE = 16*1024; + var + f : File of Byte; + locSize, locReaded, c : LongInt; + locBuffer : PByte; + locBlockSize : LongInt; + begin + Result := nil; + if (filename='') then + exit; + Assign(f,filename); + Reset(f); + if (IOResult<>0) then + begin + Close(f); + exit; + end; + locSize:=FileSize(f); + if (locSize(locSize-locReaded)) then + locBlockSize:=locSize-locReaded; + BlockRead(f,locBuffer[locReaded],locBlockSize,c); + if (IOResult<>0) or (c<=0) then + begin + FreeMem(locBuffer,locSize); + Close(f); + exit; + end; + locReaded:=locReaded+c; + end; + Result:=loadbinaryunicodemapping(locBuffer,locSize); + FreeMem(locBuffer,locSize); + Close(f); + end; + {$POP} + + procedure freemapping(amapping : punicodemap); + begin + if (amapping = nil) then + exit; + if (amapping^.map <> nil) then + freemem(amapping^.map); + if (amapping^.reversemap <> nil) then + freemem(amapping^.reversemap); + dispose(amapping); + end; + + function loadbinaryunicodemapping( + const AData : Pointer; + const ADataLength : Integer + ) : punicodemap; + var + dataPointer : PByte; + readedLength : LongInt; + + function ReadBuffer(ADest : Pointer; ALength : LongInt) : Boolean; + begin + Result := (readedLength + ALength) <= ADataLength; + if not result then + exit; + Move(dataPointer^,ADest^,ALength); + Inc(dataPointer,ALength); + readedLength := readedLength + ALength; + end; + + var + h : TSerializedMapHeader; + r : punicodemap; + begin + Result := nil; + readedLength := 0; + dataPointer := AData; + if not ReadBuffer(@h,SizeOf(h)) then + exit; + New(r); + FillChar(r^,SizeOf(tunicodemap),0); + r^.cpname := h.cpName; + r^.cp := h.cp; + r^.map := AllocMem(h.mapLength); + if not ReadBuffer(r^.map,h.mapLength) then + begin + freemapping(r); + exit; + end; + r^.lastchar := h.lastChar; + r^.reversemap := AllocMem(h.reverseMapLength); + if not ReadBuffer(r^.reversemap,h.reverseMapLength) then + begin + freemapping(r); + exit; + end; + r^.reversemaplength := (h.reverseMapLength div SizeOf(treversecharmapping)); + Result := r; + end; + procedure registermapping(p : punicodemap); begin @@ -385,6 +534,19 @@ unit charset; {$endif FPC_HAS_FEATURE_THREADING} strmapcache : string; strmapcachep : punicodemap; + + function registerbinarymapping(const directory, cpname : string) : Boolean; + var + p : punicodemap; + begin + Result := False; + p := loadbinaryunicodemapping(directory,cpname); + if (p = nil) then + exit; + registermapping(p); + Result := True; + end; + function getmap(const s : string) : punicodemap; var diff --git a/utils/creumap.pp b/utils/creumap.pp index a2629dd3f7..4a164880d4 100644 --- a/utils/creumap.pp +++ b/utils/creumap.pp @@ -26,6 +26,70 @@ program creumap; halt(1); end; +type + TEndianKind = (Little, Big); +const + ENDIAN_SUFFIX : array[TEndianKind] of string[2] = ('le','be'); +{$IFDEF ENDIAN_LITTLE} + ENDIAN_NATIVE = TEndianKind.Little; + ENDIAN_NON_NATIVE = TEndianKind.Big; +{$ENDIF ENDIAN_LITTLE} +{$IFDEF ENDIAN_BIG} + ENDIAN_NATIVE = TEndianKind.Big; + ENDIAN_NON_NATIVE = TEndianKind.Little; +{$ENDIF ENDIAN_BIG} + + procedure CreateBinaryFile(AMap : punicodemap; const ABaseFile : string); + + var + nef, oef : File of Byte; + h, th : TSerializedMapHeader; + k : Integer; + um : tunicodecharmapping; + pum : punicodecharmapping; + rm : treversecharmapping; + prm : preversecharmapping; + begin + FillChar(h,SizeOf(h),0); + h.cpName := AMap^.cpname; + h.cp := AMap^.cp; + h.lastChar := AMap^.lastchar; + h.mapLength := (AMap^.lastchar+1)*SizeOf(tunicodecharmapping); + h.reverseMapLength := AMap^.reversemaplength*SizeOf(treversecharmapping); + Assign(nef,(ABaseFile+'_'+ENDIAN_SUFFIX[ENDIAN_NATIVE]+'.bcm')); + Rewrite(nef); + BlockWrite(nef,h,SizeOf(h)); + BlockWrite(nef,AMap^.map^,h.mapLength); + BlockWrite(nef,AMap^.reversemap^,h.reverseMapLength); + Close(nef); + + th.cpName := h.cpName; + th.cp := SwapEndian(h.cp); + th.mapLength := SwapEndian(h.mapLength); + th.lastChar := SwapEndian(h.lastChar); + th.reverseMapLength := SwapEndian(h.reverseMapLength); + Assign(oef,(ABaseFile+'_'+ENDIAN_SUFFIX[ENDIAN_NON_NATIVE]+'.bcm')); + Rewrite(oef); + BlockWrite(oef,th,SizeOf(th)); + pum := AMap^.map; + for k := 0 to AMap^.lastchar do begin + um.flag := pum^.flag; + um.reserved := pum^.reserved; + um.unicode := SwapEndian(pum^.unicode); + BlockWrite(oef,um,SizeOf(um)); + Inc(pum); + end; + prm := AMap^.reversemap; + for k := 0 to AMap^.reversemaplength - 1 do begin + rm.unicode := SwapEndian(prm^.unicode); + rm.char1 := prm^.char1; + rm.char2 := prm^.char2; + BlockWrite(oef,rm,SizeOf(rm)); + Inc(prm); + end; + Close(oef); + end; + var p : punicodemap; i : longint; @@ -106,4 +170,6 @@ begin writeln(t,' registermapping(@unicodemap)'); writeln(t,' end.'); close(t); + + CreateBinaryFile(p,paramstr(1)); end.