mirror of
https://gitlab.com/freepascal.org/fpc/source.git
synced 2025-09-24 15:49:17 +02:00
* made cwstring thread safe without locks + test (twide4): widestring
manager now has two extra parameterless procedures (ThreadInitProc
and ThreadFiniProc) which are called whenever a thread begins/ends,
and cwstring uses these to create separate iconv handles for
each thread (via threadvars)
* renamed UCS4 to UCS-4BE/LE, because UCS4 is not recognised by most
systems
* clean up all iconv handles on exit, and check whether they are
valid before doing so
git-svn-id: trunk@7949 -
This commit is contained in:
Jonas Maebe
parent
5a39db65b9
commit
0d8594a705
1
.gitattributes
vendored
1
.gitattributes
vendored
@ -7016,6 +7016,7 @@ tests/test/tw6727.pp svneol=native#text/plain
|
||||
tests/test/twide1.pp svneol=native#text/plain
|
||||
tests/test/twide2.pp svneol=native#text/plain
|
||||
tests/test/twide3.pp svneol=native#text/plain
|
||||
tests/test/twide4.pp svneol=native#text/plain
|
||||
tests/test/twrstr1.pp svneol=native#text/plain
|
||||
tests/test/twrstr2.pp svneol=native#text/plain
|
||||
tests/test/twrstr3.pp svneol=native#text/plain
|
||||
|
||||
@ -30,6 +30,10 @@ Var
|
||||
{$endif HAS_MEMORYMANAGER}
|
||||
if MemoryManager.InitThread <> nil then
|
||||
MemoryManager.InitThread();
|
||||
{$ifdef FPC_HAS_FEATURE_WIDESTRINGS}
|
||||
if assigned(widestringmanager.ThreadInitProc) then
|
||||
widestringmanager.ThreadInitProc;
|
||||
{$endif FPC_HAS_FEATURE_WIDESTRINGS}
|
||||
{ ExceptAddrStack and ExceptObjectStack are threadvars }
|
||||
{ so every thread has its on exception handling capabilities }
|
||||
SysInitExceptions;
|
||||
@ -45,6 +49,10 @@ Var
|
||||
|
||||
procedure DoneThread;
|
||||
begin
|
||||
{$ifdef FPC_HAS_FEATURE_WIDESTRINGS}
|
||||
if assigned(widestringmanager.ThreadFiniProc) then
|
||||
widestringmanager.ThreadFiniProc;
|
||||
{$endif FPC_HAS_FEATURE_WIDESTRINGS}
|
||||
{$ifndef HAS_MEMORYMANAGER}
|
||||
FinalizeHeap;
|
||||
{$endif HAS_MEMORYMANAGER}
|
||||
|
||||
@ -71,6 +71,8 @@ Type
|
||||
StrLICompAnsiStringProc : function(S1, S2: PChar; MaxLen: PtrUInt): PtrInt;
|
||||
StrLowerAnsiStringProc : function(Str: PChar): PChar;
|
||||
StrUpperAnsiStringProc : function(Str: PChar): PChar;
|
||||
ThreadInitProc : procedure;
|
||||
ThreadFiniProc : procedure;
|
||||
end;
|
||||
|
||||
|
||||
|
||||
@ -44,16 +44,7 @@ Const
|
||||
libiconvname='iconv';
|
||||
{$endif}
|
||||
|
||||
{ Case-mapping "arrays" }
|
||||
var
|
||||
AnsiUpperChars: AnsiString; // 1..255
|
||||
AnsiLowerChars: AnsiString; // 1..255
|
||||
WideUpperChars: WideString; // 1..65535
|
||||
WideLowerChars: WideString; // 1..65535
|
||||
|
||||
{ the following declarations are from the libc unit for linux so they
|
||||
might be very linux centric
|
||||
maybe this needs to be splitted in an os depend way later }
|
||||
{ helper functions from libc }
|
||||
function towlower(__wc:wint_t):wint_t;cdecl;external libiconvname name 'towlower';
|
||||
function towupper(__wc:wint_t):wint_t;cdecl;external libiconvname name 'towupper';
|
||||
function wcscoll (__s1:pwchar_t; __s2:pwchar_t):cint;cdecl;external libiconvname name 'wcscoll';
|
||||
@ -92,9 +83,11 @@ const
|
||||
|
||||
{ unicode encoding name }
|
||||
{$ifdef FPC_LITTLE_ENDIAN}
|
||||
unicode_encoding = 'UTF-16LE';
|
||||
unicode_encoding2 = 'UTF-16LE';
|
||||
unicode_encoding4 = 'UCS-4LE';
|
||||
{$else FPC_LITTLE_ENDIAN}
|
||||
unicode_encoding = 'UTF-16BE';
|
||||
unicode_encoding2 = 'UTF-16BE';
|
||||
unicode_encoding4 = 'UCS-4BE';
|
||||
{$endif FPC_LITTLE_ENDIAN}
|
||||
|
||||
type
|
||||
@ -113,33 +106,11 @@ function iconv(__cd:iconv_t; __inbuf:ppchar; __inbytesleft:psize_t; __outbuf:ppc
|
||||
function iconv_close(__cd:iconv_t):cint;cdecl;external libiconvname name 'libiconv_close';
|
||||
{$endif}
|
||||
|
||||
var
|
||||
threadvar
|
||||
iconv_ansi2ucs4,
|
||||
iconv_ucs42ansi,
|
||||
iconv_ansi2wide,
|
||||
iconv_wide2ansi : iconv_t;
|
||||
|
||||
lock_ansi2ucs4 : integer = -1;
|
||||
lock_ucs42ansi : integer = -1;
|
||||
lock_ansi2wide : integer = -1;
|
||||
lock_wide2ansi : integer = -1;
|
||||
|
||||
iconv_lock : TRTLcriticalsection;
|
||||
|
||||
{
|
||||
procedure lockiconv(var lockcount: integer);
|
||||
begin
|
||||
while interlockedincrement(lockcount) <> 0 do begin
|
||||
interlockeddecrement(lockcount);
|
||||
sleep(0);
|
||||
end;
|
||||
end;
|
||||
|
||||
procedure unlockiconv(var lockcount: integer);
|
||||
begin
|
||||
interlockeddecrement(lockcount);
|
||||
end;
|
||||
}
|
||||
|
||||
procedure Wide2AnsiMove(source:pwidechar;var dest:ansistring;len:SizeInt);
|
||||
var
|
||||
@ -161,7 +132,6 @@ procedure Wide2AnsiMove(source:pwidechar;var dest:ansistring;len:SizeInt);
|
||||
srcpos:=source;
|
||||
destpos:=pchar(dest);
|
||||
outleft:=outlength;
|
||||
entercriticalsection(iconv_lock);
|
||||
while iconv(iconv_wide2ansi,ppchar(@srcpos),@srclen,@destpos,@outleft)=size_t(-1) do
|
||||
begin
|
||||
case fpgetCerrno of
|
||||
@ -187,11 +157,9 @@ procedure Wide2AnsiMove(source:pwidechar;var dest:ansistring;len:SizeInt);
|
||||
destpos:=pchar(dest)+outoffset;
|
||||
end;
|
||||
else
|
||||
leavecriticalsection(iconv_lock);
|
||||
runerror(231);
|
||||
end;
|
||||
end;
|
||||
leavecriticalsection(iconv_lock);
|
||||
// truncate string
|
||||
setlength(dest,length(dest)-outleft);
|
||||
end;
|
||||
@ -216,7 +184,6 @@ procedure Ansi2WideMove(source:pchar;var dest:widestring;len:SizeInt);
|
||||
srcpos:=source;
|
||||
destpos:=pchar(dest);
|
||||
outleft:=outlength*2;
|
||||
entercriticalsection(iconv_lock);
|
||||
while iconv(iconv_ansi2wide,@srcpos,psize(@len),@destpos,@outleft)=size_t(-1) do
|
||||
begin
|
||||
case fpgetCerrno of
|
||||
@ -242,11 +209,9 @@ procedure Ansi2WideMove(source:pchar;var dest:widestring;len:SizeInt);
|
||||
destpos:=pchar(dest)+outoffset;
|
||||
end;
|
||||
else
|
||||
leavecriticalsection(iconv_lock);
|
||||
runerror(231);
|
||||
end;
|
||||
end;
|
||||
leavecriticalsection(iconv_lock);
|
||||
// truncate string
|
||||
setlength(dest,length(dest)-outleft div 2);
|
||||
end;
|
||||
@ -291,7 +256,6 @@ procedure Ansi2UCS4Move(source:pchar;var dest:UCS4String;len:SizeInt);
|
||||
srcpos:=source;
|
||||
destpos:=pchar(dest);
|
||||
outleft:=outlength*4;
|
||||
entercriticalsection(iconv_lock);
|
||||
while iconv(iconv_ansi2ucs4,@srcpos,psize(@len),@destpos,@outleft)=size_t(-1) do
|
||||
begin
|
||||
case fpgetCerrno of
|
||||
@ -306,11 +270,9 @@ procedure Ansi2UCS4Move(source:pchar;var dest:UCS4String;len:SizeInt);
|
||||
destpos:=pchar(dest)+outoffset;
|
||||
end;
|
||||
else
|
||||
leavecriticalsection(iconv_lock);
|
||||
runerror(231);
|
||||
end;
|
||||
end;
|
||||
leavecriticalsection(iconv_lock);
|
||||
// truncate string
|
||||
setlength(dest,length(dest)-outleft div 4);
|
||||
end;
|
||||
@ -338,6 +300,28 @@ function StrCompAnsi(s1,s2 : PChar): PtrInt;
|
||||
end;
|
||||
|
||||
|
||||
procedure InitThread;
|
||||
begin
|
||||
iconv_wide2ansi:=iconv_open(nl_langinfo(CODESET),unicode_encoding2);
|
||||
iconv_ansi2wide:=iconv_open(unicode_encoding2,nl_langinfo(CODESET));
|
||||
iconv_ucs42ansi:=iconv_open(nl_langinfo(CODESET),unicode_encoding4);
|
||||
iconv_ansi2ucs4:=iconv_open(unicode_encoding4,nl_langinfo(CODESET));
|
||||
end;
|
||||
|
||||
|
||||
procedure FiniThread;
|
||||
begin
|
||||
if (iconv_wide2ansi <> iconv_t(-1)) then
|
||||
iconv_close(iconv_wide2ansi);
|
||||
if (iconv_ansi2wide <> iconv_t(-1)) then
|
||||
iconv_close(iconv_ansi2wide);
|
||||
if (iconv_ucs42ansi <> iconv_t(-1)) then
|
||||
iconv_close(iconv_ucs42ansi);
|
||||
if (iconv_ansi2ucs4 <> iconv_t(-1)) then
|
||||
iconv_close(iconv_ansi2ucs4);
|
||||
end;
|
||||
|
||||
|
||||
Procedure SetCWideStringManager;
|
||||
Var
|
||||
CWideStringManager : TWideStringManager;
|
||||
@ -369,6 +353,8 @@ begin
|
||||
StrLowerAnsiStringProc
|
||||
StrUpperAnsiStringProc
|
||||
}
|
||||
ThreadInitProc:=@InitThread;
|
||||
ThreadFiniProc:=@FiniThread;
|
||||
end;
|
||||
SetWideStringManager(CWideStringManager);
|
||||
end;
|
||||
@ -376,19 +362,15 @@ end;
|
||||
|
||||
initialization
|
||||
SetCWideStringManager;
|
||||
initcriticalsection(iconv_lock);
|
||||
|
||||
{ you have to call setlocale(LC_ALL,'') to initialise the langinfo stuff }
|
||||
{ with the information from the environment variables according to POSIX }
|
||||
{ (some OSes do this automatically, but e.g. Darwin and Solaris don't) }
|
||||
setlocale(LC_ALL,'');
|
||||
|
||||
{ init conversion tables }
|
||||
iconv_wide2ansi:=iconv_open(nl_langinfo(CODESET),unicode_encoding);
|
||||
iconv_ansi2wide:=iconv_open(unicode_encoding,nl_langinfo(CODESET));
|
||||
iconv_ucs42ansi:=iconv_open(nl_langinfo(CODESET),'UCS4');
|
||||
iconv_ansi2ucs4:=iconv_open('UCS4',nl_langinfo(CODESET));
|
||||
{ init conversion tables for main program }
|
||||
InitThread;
|
||||
finalization
|
||||
donecriticalsection(iconv_lock);
|
||||
iconv_close(iconv_ansi2wide);
|
||||
{ fini conversion tables for main program }
|
||||
FiniThread;
|
||||
end.
|
||||
|
||||
92
tests/test/twide4.pp
Normal file
92
tests/test/twide4.pp
Normal file
@ -0,0 +1,92 @@
|
||||
|
||||
{$ifdef fpc}
|
||||
{$mode objfpc}
|
||||
{$endif fpc}
|
||||
|
||||
uses
|
||||
{$ifdef unix}
|
||||
cthreads, cwstring,
|
||||
{$endif}
|
||||
Classes, SysUtils;
|
||||
|
||||
type
|
||||
tc = class(tthread)
|
||||
orgstr: ansistring;
|
||||
cnvstr: widestring;
|
||||
constructor create(const s: ansistring; const w: widestring);
|
||||
procedure execute; override;
|
||||
end;
|
||||
|
||||
const
|
||||
// string with an invalid utf-8 code sequence
|
||||
str1 = #$c1#$34'Życie'#$c1#$34' jest jak papier '#$c1#$34'toaletowy'#$c1#$34' : długie, szare i '#$c1#$34'do'#$c1#$34' dupy';
|
||||
str2 = 'Życie '#$c1#$34'jest'#$c1#$34' jak papier toaletowy : '#$c1#$34'długie'#$c1#$34', szare i do '#$c1#$34'dupy'#$c1#$34'222222222222222222222222222222222222222222222222';
|
||||
str3 = 'Życie jest '#$c1#$34'jak'#$c1#$34' papier 333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333 toaletowy : długie, '#$c1#$34'szare'#$c1#$34' i do dupy';
|
||||
str4 = 'Życie jest 4444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444 jak '#$c1#$34'papier'#$c1#$34' toaletowy : długie, szare '#$c1#$34'i'#$c1#$34' do dupy';
|
||||
count = 20000;
|
||||
|
||||
var
|
||||
wstr: widestring;
|
||||
// cnvstr: ansistring;
|
||||
error: boolean;
|
||||
|
||||
|
||||
constructor tc.create(const s: ansistring; const w: widestring);
|
||||
begin
|
||||
orgstr:=s;
|
||||
cnvstr:=w;
|
||||
inherited create(true);
|
||||
end;
|
||||
|
||||
|
||||
procedure tc.execute;
|
||||
var
|
||||
i: longint;
|
||||
w: widestring;
|
||||
begin
|
||||
for i := 1 to count do
|
||||
begin
|
||||
w:=orgstr;
|
||||
if (w<>cnvstr) then
|
||||
error:=true;
|
||||
end;
|
||||
end;
|
||||
|
||||
var
|
||||
a: array[1..4] of tc;
|
||||
w1,w2,w3,w4: widestring;
|
||||
cnvstr: ansistring;
|
||||
begin
|
||||
error:=false;
|
||||
cnvstr:=str1;
|
||||
w1:=cnvstr;
|
||||
cnvstr:=str2;
|
||||
w2:=cnvstr;
|
||||
cnvstr:=str3;
|
||||
w3:=cnvstr;
|
||||
cnvstr:=str4;
|
||||
w4:=cnvstr;
|
||||
writeln(w1);
|
||||
writeln(w2);
|
||||
writeln(w3);
|
||||
writeln(w4);
|
||||
a[1]:=tc.create(str1,w1);
|
||||
a[2]:=tc.create(str2,w2);
|
||||
a[3]:=tc.create(str3,w3);
|
||||
a[4]:=tc.create(str4,w4);
|
||||
a[1].resume;
|
||||
a[2].resume;
|
||||
a[3].resume;
|
||||
a[4].resume;
|
||||
a[1].waitfor;
|
||||
a[2].waitfor;
|
||||
a[3].waitfor;
|
||||
a[4].waitfor;
|
||||
a[1].free;
|
||||
a[2].free;
|
||||
a[3].free;
|
||||
a[4].free;
|
||||
|
||||
if error then
|
||||
halt(1);
|
||||
end.
|
||||
Loading…
Reference in New Issue
Block a user