Merged revision(s) 50764-50765 #e260cc8e86-#e260cc8e86 from trunk:

LCL: unified RST and RSJ file handling. This fixes false fuzzying multiline strings and adding empty ones to .po file when regenerating e.g. lazaruside.po from RSJ files, which are created by FPC 3.0.0 and up.
........
IDE: adapted localize scripts to work with RSJ files too
........

git-svn-id: branches/fixes_1_6@50767 -

lcl/translations.pas

@@ -1152,9 +1152,9 @@ end;
 
 procedure TPOFile.UpdateStrings(InputLines: TStrings; SType: TStringsType);
 var
-  i,j,n: integer;
+  i, j, n: integer;
   p: LongInt;
-  Identifier, Value,Line: string;
+  Identifier, Value, Line: string;
   Ch: Char;
   MultiLinedValue: boolean;
 
@@ -1170,6 +1170,41 @@ var
     p := 1;
   end;
 
+  procedure NormalizeValue;
+  begin
+    if MultiLinedValue then begin
+      // check that we end on lineending, multilined
+      // resource strings from rst usually do not end
+      // in lineending, fix here.
+      if not (Value[Length(Value)] in [#13,#10]) then
+        Value := Value + LineEnding;
+
+      //treat #10#13 sequences as #13#10 for consistency,
+      //e.g. #10#13#13#13#10#13#10 should become #13#10#13#13#10#13#10
+      p:=2;
+      while p<=Length(Value) do begin
+        if (Value[p]=#13) and (Value[p-1]=#10) then begin
+          Value[p]:=#10;
+          Value[p-1]:=#13;
+        end;
+        // further analysis shouldn't affect found #13#10 pair
+        if (Value[p]=#10) and (Value[p-1]=#13) then
+          inc(p);
+        inc(p);
+      end;
+      Value := AdjustLineBreaks(Value);
+    end;
+    // po requires special characters as #number
+    p:=1;
+    while p<=length(Value) do begin
+      j := UTF8CharacterLength(pchar(@Value[p]));
+      if (j=1) and (Value[p] in [#0..#9,#11,#12,#14..#31,#127..#255]) then
+        Value := copy(Value,1,p-1)+'#'+IntToStr(ord(Value[p]))+copy(Value,p+1,length(Value))
+      else
+        inc(p,j);
+    end;
+  end;
+
   procedure UpdateFromRsj;
   var
     Parser: TJSONParser;
@@ -1185,6 +1220,7 @@ var
         JsonItems := JsonData.Arrays['strings'];
         for K := 0 to JsonItems.Count - 1 do
         begin
+          MultiLinedValue := false;
           JsonItem := JsonItems.Items[K] as TJSONObject;
           Data:=JsonItem.Find('sourcebytes');
           if Data is TJSONArray then begin
@@ -1192,11 +1228,25 @@ var
             // while 'value' contains the string encoded as UTF16 with \u hexcodes.
             SourceBytes := TJSONArray(Data);
             SetLength(Value,SourceBytes.Count);
-            for L := 1 to length(Value) do
+            for L := 1 to length(Value) do begin
               Value[L] := chr(SourceBytes.Integers[L-1]);
-          end else
+              if Value[L] in [#13,#10] then
+                MultilinedValue := True;
+            end;
+          end else begin
             Value:=JsonItem.Get('value');
-          UpdateItem(JsonItem.Get('name'), Value);
+            // check if the value we got is multilined
+            L := 1;
+            while (L<=Length(Value)) and (MultiLinedValue = false) do begin
+              if Value[L] in [#13,#10] then
+                MultilinedValue := True;
+              inc(L);
+            end;
+          end;
+          if Value<>'' then begin
+            NormalizeValue;
+            UpdateItem(JsonItem.Get('name'), Value);
+          end;
         end;
       finally
         JsonData.Free;
@@ -1288,38 +1338,7 @@ begin
             end;
 
             if Value<>'' then begin
-              if MultiLinedValue then begin
+              NormalizeValue;
-                // check that we end on lineending, multilined
-                // resource strings from rst usually do not end
-                // in lineending, fix here.
-                if not (Value[Length(Value)] in [#13,#10]) then
-                  Value := Value + LineEnding;
-
-                //treat #10#13 sequences as #13#10 for consistency,
-                //e.g. #10#13#13#13#10#13#10 should become #13#10#13#13#10#13#10
-                p:=2;
-                while p<=Length(Value) do begin
-                  if (Value[p]=#13) and (Value[p-1]=#10) then begin
-                    Value[p]:=#10;
-                    Value[p-1]:=#13;
-                  end;
-                  // further analysis shouldn't affect found #13#10 pair
-                  if (Value[p]=#10) and (Value[p-1]=#13) then
-                    inc(p);
-                  inc(p);
-                end;
-                Value := AdjustLineBreaks(Value);
-              end;
-              // po requires special characters as #number
-              p:=1;
-              while p<=length(Value) do begin
-                j := UTF8CharacterLength(pchar(@Value[p]));
-                if (j=1) and (Value[p] in [#0..#9,#11,#12,#14..#31,#127..#255]) then
-                  Value := copy(Value,1,p-1)+'#'+IntToStr(ord(Value[p]))+copy(Value,p+1,length(Value))
-                else
-                  inc(p,j);
-              end;
-
               UpdateItem(Identifier, Value);
             end;
 

localize.bat

@@ -8,7 +8,7 @@ REM This script should be executed after adding new resource strings and after
 REM udating the translated .po files.
 REM
 REM This script
-REM - converts all compiled .rst files to .po files,
+REM - converts all compiled .rsj (.rst if .rsj is not found) files to .po files,
 REM - updates all translated xx.po files
 REM
 
@@ -31,39 +31,81 @@ if not exist tools\updatepofiles.exe goto Exit_Error
 :SkipTools
 
 echo Updating language files ...
+echo.
 
 echo on
 
-@REM IDE
+@echo Updating IDE
-@set IDE_RST=units\%ArchOsWS%\LazarusIDEStrConsts.rst
+@set IDE_RSJ=units\%ArchOsWS%\LazarusIDEStrConsts.rsj
-tools\updatepofiles %IDE_RST% languages\lazaruside.po
+@if exist %IDE_RSJ% goto IDE_update
-@if exist %IDE_RST% echo RST found
+@echo RSJ file NOT found. Searching for RST.
+@set IDE_RSJ=units\%ArchOsWS%\LazarusIDEStrConsts.rst
+@if not exist %IDE_RSJ% goto SkipIDE
+:IDE_update
+@tools\updatepofiles %IDE_RSJ% languages\lazaruside.po
+@echo Translation file %IDE_RSJ% found.
+:SkipIDE
+@echo.
 
-@REM Debugger dialogs
+@echo Updating Debugger dialogs
-@set DBGD_RST=units\%ArchOsWS%\DebuggerStrConst.rst
+@set DBGD_RSJ=units\%ArchOsWS%\DebuggerStrConst.rsj
-tools\updatepofiles %DBGD_RST% languages\debuggerstrconst.po
+@if exist %DBGD_RSJ% goto DBGD_update
-@if exist %DBGD_RST% echo RST found
+@echo RSJ file NOT found. Searching for RST.
+@set DBGD_RSJ=units\%ArchOsWS%\DebuggerStrConst.rst
+@if not exist %DBGD_RSJ% goto SkipDBGD
+:DBGD_update
+@tools\updatepofiles %DBGD_RSJ% languages\debuggerstrconst.po
+@echo Translation file %DBGD_RSJ% found.
+:SkipDBGD
+@echo.
 
-@REM LazDataDesktop
+@echo Updating LazDataDesktop
-@set LazDataDesktop_RST=tools\lazdatadesktop\lib\%ArchOS%\lazdatadeskstr.rst
+@set LazDataDesktop_RSJ=tools\lazdatadesktop\lib\%ArchOS%\lazdatadeskstr.rsj
-tools\updatepofiles %LazDataDesktop_RST% tools\lazdatadesktop\languages\lazdatadesktop.po
+@if exist %LazDataDesktop_RSJ% goto LazDataDesktop_update
-@if exist %LazDataDesktop_RST% echo RST found
+@echo RSJ file NOT found. Searching for RST.
+@set LazDataDesktop_RSJ=tools\lazdatadesktop\lib\%ArchOS%\lazdatadeskstr.rst
+@if not exist %LazDataDesktop_RSJ% goto SkipLazDataDesktop
+:LazDataDesktop_update
+@tools\updatepofiles %LazDataDesktop_RSJ% tools\lazdatadesktop\languages\lazdatadesktop.po
+@echo Translation file %LazDataDesktop_RSJ% found.
+:SkipLazDataDesktop
+@echo.
 
-@REM LazDoc
+@echo Updating LazDoc
-@set LazDoc_RST=doceditor\units\%ArchOS%\lazdemsg.rst
+@set LazDoc_RSJ=doceditor\units\%ArchOS%\lazdemsg.rsj
-tools\updatepofiles %LazDoc_RST% doceditor\languages\lazde.po
+@if exist %LazDoc_RSJ% goto LazDoc_update
-@if exist %LazDoc_RST% echo RST found
+@echo RSJ file NOT found. Searching for RST.
+@set LazDoc_RSJ=doceditor\units\%ArchOS%\lazdemsg.rst
+@if not exist %LazDoc_RSJ% goto SkipLazDoc
+:LazDoc_update
+@tools\updatepofiles %LazDoc_RSJ% doceditor\languages\lazde.po
+@echo Translation file %LazDoc_RSJ% found.
+:SkipLazDoc
+@echo.
 
-@REM LazExplorer
+@echo Updating LazExplorer
-@set LazExplorer_RST=examples\lazresexplorer\lib\%ArchOS%\reconstsunit.rst
+@set LazExplorer_RSJ=examples\lazresexplorer\lib\%ArchOS%\reconstsunit.rsj
-tools\updatepofiles %LazExplorer_RST% examples\lazresexplorer\languages\resexplorer.po
+@if exist %LazExplorer_RSJ% goto LazExplorer_update
-@if exist %LazExplorer_RST% echo RST found
+@echo RSJ file NOT found. Searching for RST.
+@set LazExplorer_RSJ=examples\lazresexplorer\lib\%ArchOS%\reconstsunit.rst
+@if not exist %LazExplorer_RSJ% goto SkipLazExplorer
+:LazExplorer_update
+@tools\updatepofiles %LazExplorer_RSJ% examples\lazresexplorer\languages\resexplorer.po
+@echo Translation file %LazExplorer_RSJ% found.
+:SkipLazExplorer
+@echo.
 
-@REM LazReport editor sample
+@echo Updating LazReport editor sample
-@set LREditor_RST=components\lazreport\samples\editor\maincalleditor.rst
+@set LREditor_RSJ=components\lazreport\samples\editor\maincalleditor.rsj
-if not exist %LREditor_RST% goto SkipLREditor
+@if exist %LREditor_RSJ% goto LREditor_update
-tools\updatepofiles %LREditor_RST% components\lazreport\samples\editor\languages\calleditorwithpkg.po
+@echo RSJ file NOT found. Searching for RST.
+@set LREditor_RSJ=components\lazreport\samples\editor\maincalleditor.rst
+@if not exist %LREditor_RSJ% goto SkipLREditor
+:LREditor_update
+@tools\updatepofiles %LREditor_RSJ% components\lazreport\samples\editor\languages\calleditorwithpkg.po
+@echo Translation file %LREditor_RSJ% found.
 :SkipLREditor
+@echo.
 
 @goto Exit
 

localize.sh

@@ -27,6 +27,12 @@ if [ "@"$FPCTARGET == "@" ]; then
   fi
 fi
 
+RSEXT="rst"
+FPCVER=`fpc -iV`
+if [ "$FPCVER" \> "2.7.0" ]; then
+  RSEXT="rsj"
+fi
+
 RSTFILES=(
   ". lazarusidestrconsts lazaruside"
   ". debuggerstrconst"
@@ -44,9 +50,9 @@ for idx in ${!RSTFILES[@]}; do
   RSTFILE=${LINE[1]}  
   POFILE=${LINE[2]:-$RSTFILE}
    
-  RST=$(find $RSTDIR -name $RSTFILE.rst)
+  RST=$(find $RSTDIR -name $RSTFILE.$RSEXT)
   if [ -n "$RST" ]; then
-    RST=`find $RSTDIR -name $RSTFILE.rst | xargs ls -1t | head -1`;
+    RST=`find $RSTDIR -name $RSTFILE.$RSEXT | xargs ls -1t | head -1`;
   
     if [ -n "$RST" ]; then
       POFileFull=$RSTDIR/languages/$POFILE.po