IDE: fpc msg parser: fix position for message identifier not found

git-svn-id: trunk@45339 -

components/codetools/basiccodetools.pas

@@ -346,6 +346,8 @@ procedure ReadRawNextPascalAtom(const Source: string;
 procedure ReadRawNextPascalAtom(var Position: PChar; out AtomStart: PChar;
    const SrcEnd: PChar = nil; NestedComments: boolean = false;
    SkipDirectives: boolean = false);
+procedure ReadPriorPascalAtom(const Source: string;
+  var Position: integer; out AtomEnd: integer; NestedComments: boolean = false);
 function ReadTilPascalBracketClose(const Source: string;
    var Position: integer; NestedComments: boolean = false): boolean;
 function GetAtomLength(p: PChar; NestedComments: boolean): integer;
@@ -2072,6 +2074,380 @@ begin
   Position:=Src;
 end;
 
+procedure ReadPriorPascalAtom(const Source: string; var Position: integer; out
+  AtomEnd: integer; NestedComments: boolean);
+var
+  CommentLvl, PrePos, OldPrePos: integer;
+  IsStringConstant: boolean;
+
+  procedure ReadStringConstantBackward;
+  var PrePos: integer;
+  begin
+    while (Position>1) do begin
+      case Source[Position-1] of
+      '''':
+        begin
+          dec(Position);
+          repeat
+            dec(Position);
+          until (Position<1) or (Source[Position] in [#0,#10,#13,'''']);
+        end;
+      '0'..'9','A'..'Z','a'..'z':
+        begin
+          // test if char constant
+          PrePos:=Position-1;
+          while (PrePos>1) and (IsHexNumberChar[Source[PrePos]]) do
+            dec(PrePos);
+          if (PrePos<1) then break;
+          if (Source[PrePos]='$') then begin
+            dec(PrePos);
+            if (PrePos<1) then break;
+          end;
+          if (Source[PrePos]='#') then
+            Position:=PrePos
+          else
+            break;
+        end;
+      else
+        break;
+      end;
+    end;
+  end;
+
+  procedure ReadBackTilCodeLineEnd;
+  begin
+    dec(Position);
+    if (Position>=1) and (Source[Position] in [#10,#13])
+    and (Source[Position+1]<>Source[Position]) then
+      dec(Position);
+
+    // read backwards till line start
+    PrePos:=Position;
+    while (PrePos>=1) and (not (Source[PrePos] in [#10,#13])) do
+      dec(PrePos);
+    // read line forward to find out,
+    // if line ends in comment or string constant
+    IsStringConstant:=false;
+    repeat
+      inc(PrePos);
+      case Source[PrePos] of
+
+      '/':
+        if Source[PrePos+1]='/' then begin
+          // this was a delphi comment -> skip comment
+          Position:=PrePos-1;
+          break;
+        end;
+
+      '{':
+        begin
+          inc(PrePos);
+          if (PrePos<=Position) and (Source[PrePos]=#3) then begin
+            // skip codetools comment
+            inc(PrePos);
+            while (PrePos<=Position) do begin
+              if (Source[PrePos]=#3) and (PrePos<Position)
+              and (Source[PrePos+1]='}') then begin
+                inc(PrePos,2);
+                break;
+              end;
+              inc(PrePos);
+            end;
+          end else begin
+            // skip pascal comment
+            CommentLvl:=1;
+            while (PrePos<=Position) do begin
+              case Source[PrePos] of
+              '{': if NestedComments then inc(CommentLvl);
+              '}':
+                begin
+                  dec(CommentLvl);
+                  if CommentLvl=0 then break;
+                end;
+              end;
+              inc(PrePos);
+            end;
+          end;
+        end;
+
+      '(':
+        if Source[PrePos+1]='*' then begin
+          // skip turbo pascal comment
+          inc(PrePos,2);
+          while (PrePos<Position)
+          and ((Source[PrePos]<>'*') or (Source[PrePos+1]<>')')) do
+            inc(PrePos);
+          inc(PrePos);
+        end;
+
+      '''':
+        begin
+          // a string constant -> skip it
+          OldPrePos:=PrePos;
+          while (PrePos<Position) do begin
+            inc(PrePos);
+            case Source[PrePos] of
+            '''':
+              break;
+
+            #0,#10,#13:
+              begin
+                // string constant right border is the line end
+                // -> last atom of line found
+                IsStringConstant:=true;
+                break;
+              end;
+
+            end;
+          end;
+          if IsStringConstant then break;
+        end;
+
+      #10,#13:
+        // no comment and no string constant found
+        break;
+
+      end;
+    until PrePos>=Position;
+  end;
+
+type
+  TNumberType = (ntDecimal, ntHexadecimal, ntBinary, ntIdentifier,
+    ntCharConstant, ntFloat, ntFloatWithExponent);
+  TNumberTypes = set of TNumberType;
+
+const
+  AllNumberTypes: TNumberTypes = [ntDecimal, ntHexadecimal, ntBinary,
+    ntIdentifier, ntCharConstant, ntFloat, ntFloatWithExponent];
+
+var c1, c2: char;
+  ForbiddenNumberTypes: TNumberTypes;
+begin
+  // Skip all spaces and comments
+  CommentLvl:=0;
+  dec(Position);
+  IsStringConstant:=false;
+  OldPrePos:=0;
+  while Position>=1 do begin
+    if IsCommentEndChar[Source[Position]] then begin
+      case Source[Position] of
+
+      '}':
+        begin
+          dec(Position);
+          if (Position>=1) and (Source[Position]=#3) then begin
+            // codetools skip comment {#3 #3}
+            dec(Position);
+            while (Position>=1) do begin
+              if (Source[Position]=#3) and (Position>1)
+              and (Source[Position-1]='}') then begin
+                dec(Position,2);
+                break;
+              end;
+              dec(Position);
+            end;
+          end else begin
+            // pascal comment {}
+            CommentLvl:=1;
+            while (Position>=1) and (CommentLvl>0) do begin
+              case Source[Position] of
+              '}': if NestedComments then inc(CommentLvl);
+              '{': dec(CommentLvl);
+              end;
+              dec(Position);
+            end;
+          end;
+        end;
+
+      #10,#13: // possible Delphi comment
+        ReadBackTilCodeLineEnd;
+
+      ')': // old turbo pascal comment
+        if (Position>1) and (Source[Position-1]='*') then begin
+          dec(Position,3);
+          while (Position>=1)
+          and ((Source[Position]<>'(') or (Source[Position+1]<>'*')) do
+            dec(Position);
+          dec(Position);
+        end else
+          break;
+
+      end;
+    end else if IsSpaceChar[Source[Position]] then begin
+      repeat
+        dec(Position);
+      until (Position<1) or (Source[Position] in [#10,#13])
+      or (not (IsSpaceChar[Source[Position]]));
+    end else begin
+      break;
+    end;
+  end;
+  // Position now points to the last char of the prior atom
+  AtomEnd:=Position+1;
+  if Position<1 then begin
+    Position:=1;
+    AtomEnd:=1;
+    exit;
+  end;
+  // read atom
+  if IsStringConstant then begin
+    Position:=OldPrePos;
+    if (Position>1) and (Source[Position-1]='''') then begin
+      ReadStringConstantBackward;
+    end;
+    exit;
+  end;
+  c2:=Source[Position];
+  case c2 of
+    '_','A'..'Z','a'..'z':
+      begin
+        // identifier or keyword or hexnumber
+        while (Position>1) do begin
+          if (IsIdentChar[Source[Position-1]]) then
+            dec(Position)
+          else begin
+            case UpChars[Source[Position-1]] of
+            '@':
+              // assembler label
+              if (Position>2)
+              and (Source[Position-2]='@') then
+                dec(Position,2);
+            '$':
+              // hex number
+              dec(Position);
+            end;
+            break;
+          end;
+        end;
+      end;
+    '''':
+      begin
+        inc(Position);
+        ReadStringConstantBackward;
+      end;
+    '0'..'9':
+      begin
+        // could be a decimal number, an identifier, a hex number,
+        // a binary number, a char constant, a float, a float with exponent
+        ForbiddenNumberTypes:=[];
+        while true do begin
+          case UpChars[Source[Position]] of
+          '0'..'1':
+            ;
+          '2'..'9':
+            ForbiddenNumberTypes:=ForbiddenNumberTypes+[ntBinary];
+          'A'..'D','F':
+            ForbiddenNumberTypes:=ForbiddenNumberTypes
+               +[ntBinary,ntDecimal,ntCharConstant,ntFloat,ntFloatWithExponent];
+          'E':
+            ForbiddenNumberTypes:=ForbiddenNumberTypes
+               +[ntBinary,ntDecimal,ntCharConstant,ntFloat];
+          'G'..'Z','_':
+            ForbiddenNumberTypes:=AllNumberTypes-[ntIdentifier];
+          '.':
+            begin
+              // could be the point of a float
+              if (ntFloat in ForbiddenNumberTypes)
+              or (Position<=1) or (Source[Position-1]='.') then begin
+                inc(Position);
+                break;
+              end;
+              dec(Position);
+              // this was the part of a float after the point
+              //  -> read decimal in front
+              ForbiddenNumberTypes:=AllNumberTypes-[ntDecimal];
+            end;
+          '+','-':
+            begin
+              // could be part of an exponent
+              if (ntFloatWithExponent in ForbiddenNumberTypes)
+              or (Position<=1)
+              or (not (Source[Position-1] in ['e','E']))
+              then begin
+                inc(Position);
+                break;
+              end;
+              dec(Position);
+              // this was the exponent of a float -> read the float
+              ForbiddenNumberTypes:=AllNumberTypes-[ntFloat];
+            end;
+          '#': // char constant found
+            begin
+              if (ntCharConstant in ForbiddenNumberTypes) then
+                inc(Position);
+              ReadStringConstantBackward;
+              break;
+            end;
+          '$':
+            begin
+              // hexadecimal number found
+              if (ntHexadecimal in ForbiddenNumberTypes) then
+                inc(Position);
+              break;
+            end;
+          '%':
+            begin
+              // binary number found
+              if (ntBinary in ForbiddenNumberTypes) then
+                inc(Position);
+              break;
+            end;
+          '@':
+            begin
+              if (Position=1) or (Source[Position-1]<>'@')
+              or (([ntIdentifier,ntDecimal]*ForbiddenNumberTypes)=[]) then
+                // atom start found
+                inc(Position)
+              else
+                // label found
+                dec(Position);
+              break;
+            end;
+          else
+            begin
+              inc(Position);
+              break;
+            end;
+          end;
+          if ForbiddenNumberTypes=AllNumberTypes then begin
+            inc(Position);
+            break;
+          end;
+          if Position<=1 then break;
+          dec(Position);
+        end;
+        if IsIdentStartChar[Source[Position]] then begin
+          // it is an identifier
+        end;
+      end;
+
+    ';': ;
+    ':': ;
+    ',': ;
+    '(': ;
+    ')': ;
+    '[': ;
+    ']': ;
+
+    else
+      begin
+        if Position>1 then begin
+          c1:=Source[Position-1];
+          // test for double char operators :=, +=, -=, /=, *=, <>, <=, >=, **, ><
+          if ((c2='=') and  (IsEqualOperatorStartChar[c1]))
+          or ((c1='<') and (c2='>'))
+          or ((c1='>') and (c2='<'))
+          or ((c1='.') and (c2='.'))
+          or ((c1='*') and (c2='*'))
+          or ((c1='@') and (c2='@'))
+          then begin
+            dec(Position);
+          end;
+        end;
+      end;
+  end;
+end;
+
 function ReadTilPascalBracketClose(const Source: string; var Position: integer;
   NestedComments: boolean): boolean;
 // Input: Position points right after the opening bracket

components/codetools/sourcelog.pas

@@ -135,7 +135,7 @@ type
     function LineCount: integer;
     function GetLine(Index: integer): string; // 0-based
     function GetLineLength(Index: integer): integer; // 0-based
-    procedure GetLineRange(Index: integer; out LineRange: TLineRange);
+    procedure GetLineRange(Index: integer; out LineRange: TLineRange); // 0-based
     function GetLineStart(Index: integer): integer; // 1-based
     property Items[Index: integer]: TSourceLogEntry
        read GetItems write SetItems; default;

ide/etfpcmsgparser.pas

@@ -34,11 +34,12 @@ unit etFPCMsgParser;
 interface
 
 uses
-  Classes, SysUtils, strutils, FileProcs, KeywordFuncLists, IDEExternToolIntf,
+  Classes, SysUtils, strutils, math, FileProcs, KeywordFuncLists,
-  PackageIntf, LazIDEIntf, ProjectIntf, IDEUtils, CompOptsIntf,
+  IDEExternToolIntf, PackageIntf, LazIDEIntf, ProjectIntf, IDEUtils,
-  CodeToolsFPCMsgs, CodeToolsStructs, CodeCache, CodeToolManager,
+  CompOptsIntf, CodeToolsFPCMsgs, CodeToolsStructs, CodeCache, CodeToolManager,
-  DirectoryCacher, BasicCodeTools, DefineTemplates, LazUTF8, FileUtil,
+  DirectoryCacher, BasicCodeTools, DefineTemplates, SourceLog, LazUTF8,
-  LConvEncoding, TransferMacros, etMakeMsgParser, EnvironmentOpts;
+  FileUtil, LConvEncoding, TransferMacros, etMakeMsgParser, EnvironmentOpts,
+  LCLProc;
 
 const
   FPCMsgIDLogo = 11023;
@@ -141,22 +142,22 @@ type
 
   TIDEFPCParser = class(TFPCParser)
   private
-    fMsgID: Integer; // current message id given by ReadLine (-vq)
+    fCurSource: TCodeBuffer;
-    fOutputIndex: integer; // current OutputIndex given by ReadLine
-    fLineToMsgID: TPatternToMsgIDs;
-    fLastWorkerImprovedMessage: array[boolean] of integer;
-    fLastSource: TCodeBuffer;
     fFileExists: TFilenameToPointerTree;
-    fIncludePathValidForWorkerDir: string;
     fIncludePath: string; // only valid if fIncludePathValidForWorkerDir=Tool.WorkerDirectory
-    fMsgItemUnitNotUsed: TFPCMsgItem;
+    fIncludePathValidForWorkerDir: string;
+    fLastWorkerImprovedMessage: array[boolean] of integer;
+    fLineToMsgID: TPatternToMsgIDs;
+    fMissingFPCMsgItem: TFPCMsgItem;
+    fMsgID: Integer; // current message id given by ReadLine (-vq)
     fMsgItemCantFindUnitUsedBy: TFPCMsgItem;
     fMsgItemCompilationAborted: TFPCMsgItem;
-    fMsgItemThereWereErrorsCompiling: TFPCMsgItem;
-    fMsgItemIdentifierNotFound: TFPCMsgItem;
-    fMsgItemErrorWhileLinking: TFPCMsgItem;
     fMsgItemErrorWhileCompilingResources: TFPCMsgItem;
-    fMissingFPCMsgItem: TFPCMsgItem;
+    fMsgItemErrorWhileLinking: TFPCMsgItem;
+    fMsgItemIdentifierNotFound: TFPCMsgItem;
+    fMsgItemThereWereErrorsCompiling: TFPCMsgItem;
+    fMsgItemUnitNotUsed: TFPCMsgItem;
+    fOutputIndex: integer; // current OutputIndex given by ReadLine
     function FileExists(const Filename: string; aSynchronized: boolean): boolean;
     function CheckForMsgId(p: PChar): boolean; // (MsgId) message
     function CheckForFileLineColMessage(p: PChar): boolean; // the normal messages: filename(y,x): Hint: ..
@@ -181,6 +182,8 @@ type
       MsgLine: TMessageLine);
     procedure ImproveMsgLinkerUndefinedReference(aSynchronized: boolean;
       MsgLine: TMessageLine);
+    procedure ImproveMsgIdentifierPosition(SourceOK: boolean; aSynchronized: boolean;
+      MsgLine: TMessageLine);
     procedure Translate(p: PChar; MsgItem, TranslatedItem: TFPCMsgItem;
       out TranslatedMsg: String; out MsgType: TMessageLineUrgency);
     function LongenFilename(MsgLine: TMessageLine; aFilename: string): string; // (worker thread)
@@ -1012,7 +1015,7 @@ destructor TIDEFPCParser.Destroy;
 begin
   FreeAndNil(FFilesToIgnoreUnitNotUsed);
   FreeAndNil(fFileExists);
-  FreeAndNil(fLastSource);
+  FreeAndNil(fCurSource);
   if TranslationFile<>nil then
     FPCMsgFilePool.UnloadFile(TranslationFile);
   if MsgFile<>nil then
@@ -1107,7 +1110,7 @@ end;
 
 procedure TIDEFPCParser.Done;
 begin
-  FreeAndNil(fLastSource);
+  FreeAndNil(fCurSource);
   inherited Done;
 end;
 
@@ -1554,10 +1557,10 @@ begin
   begin
     X:=MsgLine.Column;
     Y:=MsgLine.Line;
-    if (y<=fLastSource.LineCount) and (x-1<=fLastSource.GetLineLength(y-1))
+    if (y<=fCurSource.LineCount) and (x-1<=fCurSource.GetLineLength(y-1))
     then begin
-      p:=PChar(fLastSource.Source)+fLastSource.GetLineStart(y-1)+x-2;
+      p:=PChar(fCurSource.Source)+fCurSource.GetLineStart(y-1)+x-2;
-      //debugln(['TFPCParser.ImproveMessages ',aFilename,' ',Y,',',X,' ',copy(fLastSource.GetLine(y-1),1,x-1),'|',copy(fLastSource.GetLine(y-1),x,100),' p=',p[0],p[1],p[2]]);
+      //debugln(['TFPCParser.ImproveMessages ',aFilename,' ',Y,',',X,' ',copy(fCurSource.GetLine(y-1),1,x-1),'|',copy(fCurSource.GetLine(y-1),x,100),' p=',p[0],p[1],p[2]]);
       if ((p^='{') and (p[1]='%') and (p[2]='H') and (p[3]='-'))
       or ((x>5) and (p[-5]='{') and (p[-4]='%') and (p[-3]='H') and (p[-2]='-')
         and (p[-1]='}'))
@@ -1911,6 +1914,59 @@ begin
   if CheckForFileAndLineNumber then exit;
 end;
 
+procedure TIDEFPCParser.ImproveMsgIdentifierPosition(SourceOK: boolean;
+  aSynchronized: boolean; MsgLine: TMessageLine);
+{ FPC report the token after the identifier
+  => fix the position
+  For example:
+    unit1.pas(42,5) Error: (5000) Identifier not found "i"
+    "  i :="
+}
+var
+  LineRange: TLineRange;
+  Line, Col: Integer;
+  p, AtomEnd: integer;
+  Src: String;
+  Identifier: String;
+begin
+  if (not IsMsgID(MsgLine,FPCMsgIDIdentifierNotFound,fMsgItemIdentifierNotFound))
+  or (MsgLine.Column<1) or (MsgLine.Line<1) then
+    exit;
+  if (MsgLine.Line=1) and (MsgLine.Column=1) then exit;
+  if (not SourceOK) then begin
+    if (not aSynchronized) then
+      NeedSynchronize:=true;
+    exit;
+  end;
+  Line:=MsgLine.Line;
+  //DebuglnThreadLog(['Old Line=',Line,' ',MsgLine.Column]);
+  if Line>=fCurSource.LineCount then exit;
+  Identifier:=GetFPCMsgValue1(MsgLine);
+  if Identifier='' then exit;
+  fCurSource.GetLineRange(Line-1,LineRange);
+  //DebuglnThreadLog(['Old Range=',LineRange.StartPos,'-',LineRange.EndPos,' Str="',copy(fCurSource.Source,LineRange.StartPos,LineRange.EndPos-LineRange.StartPos),'"']);
+  Col:=Min(MsgLine.Column,LineRange.EndPos-LineRange.StartPos+1);
+  p:=LineRange.StartPos+Col-1;
+  Src:=fCurSource.Source;
+  if CompareIdentifiers(PChar(Identifier),@Src[p])=0 then begin
+    // already pointing at the right identifier
+    exit;
+  end;
+  // go to prior token
+  //DebuglnThreadLog(['New Line=',Line,' Col=',Col,' p=',p]);
+  ReadPriorPascalAtom(Src,p,AtomEnd,false);
+  if p<1 then exit;
+  if CompareIdentifiers(PChar(Identifier),@Src[p])<>0 then begin
+    // the prior token is not the identifier neither
+    // => don't know
+    exit;
+  end;
+  fCurSource.AbsoluteToLineCol(p,Line,Col);
+  //DebuglnThreadLog(['New Line=',Line,' Col=',Col,' p=',p]);
+  if (Line<1) or (Col<1) then exit;
+  MsgLine.SetSourcePosition(MsgLine.Filename,Line,Col)
+end;
+
 procedure TIDEFPCParser.Translate(p: PChar; MsgItem, TranslatedItem: TFPCMsgItem;
   out TranslatedMsg: String; out MsgType: TMessageLineUrgency);
 begin
@@ -1923,7 +1979,7 @@ begin
   if TranslatedItem<>nil then begin
     if System.Pos('$',TranslatedItem.Pattern)<1 then begin
       TranslatedMsg:=TranslatedItem.Pattern;
-      UTF8FixBroken(TranslatedMsg);
+      LazUTF8.UTF8FixBroken(TranslatedMsg);
     end
     else if MsgItem<>nil then
       TranslatedMsg:=TranslateFPCMsg(p,MsgItem.Pattern,TranslatedItem.Pattern);
@@ -2338,8 +2394,8 @@ begin
       SourceOK:=false;
       aFilename:=MsgLine.Filename;
       if FilenameIsAbsolute(aFilename) then begin
-        if (fLastSource<>nil)
+        if (fCurSource<>nil)
-        and (CompareFilenames(aFilename,fLastSource.Filename)=0) then begin
+        and (CompareFilenames(aFilename,fCurSource.Filename)=0) then begin
           SourceOK:=true;
         end else begin
           if aSynchronized then begin
@@ -2347,10 +2403,10 @@ begin
             //debugln(['TFPCParser.ImproveMessages loading ',aFilename]);
             Code:=CodeToolBoss.LoadFile(aFilename,true,false);
             if Code<>nil then begin
-              if fLastSource=nil then
+              if fCurSource=nil then
-                fLastSource:=TCodeBuffer.Create;
+                fCurSource:=TCodeBuffer.Create;
-              fLastSource.Filename:=aFilename;
+              fCurSource.Filename:=aFilename;
-              fLastSource.Source:=Code.Source;
+              fCurSource.Source:=Code.Source;
               SourceOK:=true;
             end;
           end else begin
@@ -2364,6 +2420,7 @@ begin
       ImproveMsgUnitNotFound(aSynchronized, MsgLine);
       ImproveMsgUnitNotUsed(aSynchronized, MsgLine);
       ImproveMsgSenderNotUsed(aSynchronized, MsgLine);
+      ImproveMsgIdentifierPosition(SourceOK, aSynchronized, MsgLine);
     end else if MsgLine.SubTool=SubToolFPCLinker then begin
       ImproveMsgLinkerUndefinedReference(aSynchronized, MsgLine);
     end;