From 0d8594a705099fcc77d6964f09e12e7f524dd1ff Mon Sep 17 00:00:00 2001 From: Jonas Maebe Date: Wed, 4 Jul 2007 18:10:34 +0000 Subject: [PATCH] * made cwstring thread safe without locks + test (twide4): widestring manager now has two extra parameterless procedures (ThreadInitProc and ThreadFiniProc) which are called whenever a thread begins/ends, and cwstring uses these to create separate iconv handles for each thread (via threadvars) * renamed UCS4 to UCS-4BE/LE, because UCS4 is not recognised by most systems * clean up all iconv handles on exit, and check whether they are valid before doing so git-svn-id: trunk@7949 - --- .gitattributes | 1 + rtl/inc/thread.inc | 8 ++++ rtl/inc/wstringh.inc | 2 + rtl/unix/cwstring.pp | 86 ++++++++++++++++------------------------- tests/test/twide4.pp | 92 ++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 137 insertions(+), 52 deletions(-) create mode 100644 tests/test/twide4.pp diff --git a/.gitattributes b/.gitattributes index 2d8d6c50f6..990072235c 100644 --- a/.gitattributes +++ b/.gitattributes @@ -7016,6 +7016,7 @@ tests/test/tw6727.pp svneol=native#text/plain tests/test/twide1.pp svneol=native#text/plain tests/test/twide2.pp svneol=native#text/plain tests/test/twide3.pp svneol=native#text/plain +tests/test/twide4.pp svneol=native#text/plain tests/test/twrstr1.pp svneol=native#text/plain tests/test/twrstr2.pp svneol=native#text/plain tests/test/twrstr3.pp svneol=native#text/plain diff --git a/rtl/inc/thread.inc b/rtl/inc/thread.inc index e2aac7ebac..df0bf9b091 100644 --- a/rtl/inc/thread.inc +++ b/rtl/inc/thread.inc @@ -30,6 +30,10 @@ Var {$endif HAS_MEMORYMANAGER} if MemoryManager.InitThread <> nil then MemoryManager.InitThread(); +{$ifdef FPC_HAS_FEATURE_WIDESTRINGS} + if assigned(widestringmanager.ThreadInitProc) then + widestringmanager.ThreadInitProc; +{$endif FPC_HAS_FEATURE_WIDESTRINGS} { ExceptAddrStack and ExceptObjectStack are threadvars } { so every thread has its on exception handling capabilities } SysInitExceptions; @@ -45,6 +49,10 @@ Var procedure DoneThread; begin +{$ifdef FPC_HAS_FEATURE_WIDESTRINGS} + if assigned(widestringmanager.ThreadFiniProc) then + widestringmanager.ThreadFiniProc; +{$endif FPC_HAS_FEATURE_WIDESTRINGS} {$ifndef HAS_MEMORYMANAGER} FinalizeHeap; {$endif HAS_MEMORYMANAGER} diff --git a/rtl/inc/wstringh.inc b/rtl/inc/wstringh.inc index 784802654e..98272fec15 100644 --- a/rtl/inc/wstringh.inc +++ b/rtl/inc/wstringh.inc @@ -71,6 +71,8 @@ Type StrLICompAnsiStringProc : function(S1, S2: PChar; MaxLen: PtrUInt): PtrInt; StrLowerAnsiStringProc : function(Str: PChar): PChar; StrUpperAnsiStringProc : function(Str: PChar): PChar; + ThreadInitProc : procedure; + ThreadFiniProc : procedure; end; diff --git a/rtl/unix/cwstring.pp b/rtl/unix/cwstring.pp index e686ee4506..e9c47ea388 100644 --- a/rtl/unix/cwstring.pp +++ b/rtl/unix/cwstring.pp @@ -44,16 +44,7 @@ Const libiconvname='iconv'; {$endif} -{ Case-mapping "arrays" } -var - AnsiUpperChars: AnsiString; // 1..255 - AnsiLowerChars: AnsiString; // 1..255 - WideUpperChars: WideString; // 1..65535 - WideLowerChars: WideString; // 1..65535 - -{ the following declarations are from the libc unit for linux so they - might be very linux centric - maybe this needs to be splitted in an os depend way later } +{ helper functions from libc } function towlower(__wc:wint_t):wint_t;cdecl;external libiconvname name 'towlower'; function towupper(__wc:wint_t):wint_t;cdecl;external libiconvname name 'towupper'; function wcscoll (__s1:pwchar_t; __s2:pwchar_t):cint;cdecl;external libiconvname name 'wcscoll'; @@ -92,9 +83,11 @@ const { unicode encoding name } {$ifdef FPC_LITTLE_ENDIAN} - unicode_encoding = 'UTF-16LE'; + unicode_encoding2 = 'UTF-16LE'; + unicode_encoding4 = 'UCS-4LE'; {$else FPC_LITTLE_ENDIAN} - unicode_encoding = 'UTF-16BE'; + unicode_encoding2 = 'UTF-16BE'; + unicode_encoding4 = 'UCS-4BE'; {$endif FPC_LITTLE_ENDIAN} type @@ -113,33 +106,11 @@ function iconv(__cd:iconv_t; __inbuf:ppchar; __inbytesleft:psize_t; __outbuf:ppc function iconv_close(__cd:iconv_t):cint;cdecl;external libiconvname name 'libiconv_close'; {$endif} -var +threadvar iconv_ansi2ucs4, iconv_ucs42ansi, iconv_ansi2wide, iconv_wide2ansi : iconv_t; - - lock_ansi2ucs4 : integer = -1; - lock_ucs42ansi : integer = -1; - lock_ansi2wide : integer = -1; - lock_wide2ansi : integer = -1; - - iconv_lock : TRTLcriticalsection; - -{ -procedure lockiconv(var lockcount: integer); -begin - while interlockedincrement(lockcount) <> 0 do begin - interlockeddecrement(lockcount); - sleep(0); - end; -end; - -procedure unlockiconv(var lockcount: integer); -begin - interlockeddecrement(lockcount); -end; -} procedure Wide2AnsiMove(source:pwidechar;var dest:ansistring;len:SizeInt); var @@ -161,7 +132,6 @@ procedure Wide2AnsiMove(source:pwidechar;var dest:ansistring;len:SizeInt); srcpos:=source; destpos:=pchar(dest); outleft:=outlength; - entercriticalsection(iconv_lock); while iconv(iconv_wide2ansi,ppchar(@srcpos),@srclen,@destpos,@outleft)=size_t(-1) do begin case fpgetCerrno of @@ -187,11 +157,9 @@ procedure Wide2AnsiMove(source:pwidechar;var dest:ansistring;len:SizeInt); destpos:=pchar(dest)+outoffset; end; else - leavecriticalsection(iconv_lock); runerror(231); end; end; - leavecriticalsection(iconv_lock); // truncate string setlength(dest,length(dest)-outleft); end; @@ -216,7 +184,6 @@ procedure Ansi2WideMove(source:pchar;var dest:widestring;len:SizeInt); srcpos:=source; destpos:=pchar(dest); outleft:=outlength*2; - entercriticalsection(iconv_lock); while iconv(iconv_ansi2wide,@srcpos,psize(@len),@destpos,@outleft)=size_t(-1) do begin case fpgetCerrno of @@ -242,11 +209,9 @@ procedure Ansi2WideMove(source:pchar;var dest:widestring;len:SizeInt); destpos:=pchar(dest)+outoffset; end; else - leavecriticalsection(iconv_lock); runerror(231); end; end; - leavecriticalsection(iconv_lock); // truncate string setlength(dest,length(dest)-outleft div 2); end; @@ -291,7 +256,6 @@ procedure Ansi2UCS4Move(source:pchar;var dest:UCS4String;len:SizeInt); srcpos:=source; destpos:=pchar(dest); outleft:=outlength*4; - entercriticalsection(iconv_lock); while iconv(iconv_ansi2ucs4,@srcpos,psize(@len),@destpos,@outleft)=size_t(-1) do begin case fpgetCerrno of @@ -306,11 +270,9 @@ procedure Ansi2UCS4Move(source:pchar;var dest:UCS4String;len:SizeInt); destpos:=pchar(dest)+outoffset; end; else - leavecriticalsection(iconv_lock); runerror(231); end; end; - leavecriticalsection(iconv_lock); // truncate string setlength(dest,length(dest)-outleft div 4); end; @@ -338,6 +300,28 @@ function StrCompAnsi(s1,s2 : PChar): PtrInt; end; +procedure InitThread; +begin + iconv_wide2ansi:=iconv_open(nl_langinfo(CODESET),unicode_encoding2); + iconv_ansi2wide:=iconv_open(unicode_encoding2,nl_langinfo(CODESET)); + iconv_ucs42ansi:=iconv_open(nl_langinfo(CODESET),unicode_encoding4); + iconv_ansi2ucs4:=iconv_open(unicode_encoding4,nl_langinfo(CODESET)); +end; + + +procedure FiniThread; +begin + if (iconv_wide2ansi <> iconv_t(-1)) then + iconv_close(iconv_wide2ansi); + if (iconv_ansi2wide <> iconv_t(-1)) then + iconv_close(iconv_ansi2wide); + if (iconv_ucs42ansi <> iconv_t(-1)) then + iconv_close(iconv_ucs42ansi); + if (iconv_ansi2ucs4 <> iconv_t(-1)) then + iconv_close(iconv_ansi2ucs4); +end; + + Procedure SetCWideStringManager; Var CWideStringManager : TWideStringManager; @@ -369,6 +353,8 @@ begin StrLowerAnsiStringProc StrUpperAnsiStringProc } + ThreadInitProc:=@InitThread; + ThreadFiniProc:=@FiniThread; end; SetWideStringManager(CWideStringManager); end; @@ -376,19 +362,15 @@ end; initialization SetCWideStringManager; - initcriticalsection(iconv_lock); { you have to call setlocale(LC_ALL,'') to initialise the langinfo stuff } { with the information from the environment variables according to POSIX } { (some OSes do this automatically, but e.g. Darwin and Solaris don't) } setlocale(LC_ALL,''); - { init conversion tables } - iconv_wide2ansi:=iconv_open(nl_langinfo(CODESET),unicode_encoding); - iconv_ansi2wide:=iconv_open(unicode_encoding,nl_langinfo(CODESET)); - iconv_ucs42ansi:=iconv_open(nl_langinfo(CODESET),'UCS4'); - iconv_ansi2ucs4:=iconv_open('UCS4',nl_langinfo(CODESET)); + { init conversion tables for main program } + InitThread; finalization - donecriticalsection(iconv_lock); - iconv_close(iconv_ansi2wide); + { fini conversion tables for main program } + FiniThread; end. diff --git a/tests/test/twide4.pp b/tests/test/twide4.pp new file mode 100644 index 0000000000..fec2bab611 --- /dev/null +++ b/tests/test/twide4.pp @@ -0,0 +1,92 @@ + +{$ifdef fpc} +{$mode objfpc} +{$endif fpc} + +uses +{$ifdef unix} + cthreads, cwstring, +{$endif} + Classes, SysUtils; + +type + tc = class(tthread) + orgstr: ansistring; + cnvstr: widestring; + constructor create(const s: ansistring; const w: widestring); + procedure execute; override; + end; + +const + // string with an invalid utf-8 code sequence + str1 = #$c1#$34'Życie'#$c1#$34' jest jak papier '#$c1#$34'toaletowy'#$c1#$34' : długie, szare i '#$c1#$34'do'#$c1#$34' dupy'; + str2 = 'Życie '#$c1#$34'jest'#$c1#$34' jak papier toaletowy : '#$c1#$34'długie'#$c1#$34', szare i do '#$c1#$34'dupy'#$c1#$34'222222222222222222222222222222222222222222222222'; + str3 = 'Życie jest '#$c1#$34'jak'#$c1#$34' papier 333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333 toaletowy : długie, '#$c1#$34'szare'#$c1#$34' i do dupy'; + str4 = 'Życie jest 4444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444 jak '#$c1#$34'papier'#$c1#$34' toaletowy : długie, szare '#$c1#$34'i'#$c1#$34' do dupy'; + count = 20000; + +var + wstr: widestring; +// cnvstr: ansistring; + error: boolean; + + +constructor tc.create(const s: ansistring; const w: widestring); +begin + orgstr:=s; + cnvstr:=w; + inherited create(true); +end; + + +procedure tc.execute; +var + i: longint; + w: widestring; +begin + for i := 1 to count do + begin + w:=orgstr; + if (w<>cnvstr) then + error:=true; + end; +end; + +var + a: array[1..4] of tc; + w1,w2,w3,w4: widestring; + cnvstr: ansistring; +begin + error:=false; + cnvstr:=str1; + w1:=cnvstr; + cnvstr:=str2; + w2:=cnvstr; + cnvstr:=str3; + w3:=cnvstr; + cnvstr:=str4; + w4:=cnvstr; + writeln(w1); + writeln(w2); + writeln(w3); + writeln(w4); + a[1]:=tc.create(str1,w1); + a[2]:=tc.create(str2,w2); + a[3]:=tc.create(str3,w3); + a[4]:=tc.create(str4,w4); + a[1].resume; + a[2].resume; + a[3].resume; + a[4].resume; + a[1].waitfor; + a[2].waitfor; + a[3].waitfor; + a[4].waitfor; + a[1].free; + a[2].free; + a[3].free; + a[4].free; + + if error then + halt(1); +end.