From 258955d9062c24ec72c53827ce0d766dab441db5 Mon Sep 17 00:00:00 2001 From: DomingoGP Date: Sun, 15 Sep 2024 14:40:50 +0200 Subject: [PATCH] Jedi code format: implement special comment //jcf:parse=off/on. Related to issue #41144 --- components/jcf2/Parse/BuildTokenList.pas | 39 ++++++++++++++++++- components/jcf2/Process/FormatFlags.pas | 32 +++++++++------ .../jcf2/Process/Obfuscate/RemoveComment.pas | 6 +-- components/jcf2/Process/SwitchableVisitor.pas | 16 +++++++- components/jcf2/ReadWrite/Converter.pas | 4 ++ components/jcf2/docs/special_comments.md | 25 ++++++------ components/jcf2/jcfbaseconsts.pas | 2 + 7 files changed, 95 insertions(+), 29 deletions(-) diff --git a/components/jcf2/Parse/BuildTokenList.pas b/components/jcf2/Parse/BuildTokenList.pas index 1651953769..388cfb6b6a 100644 --- a/components/jcf2/Parse/BuildTokenList.pas +++ b/components/jcf2/Parse/BuildTokenList.pas @@ -44,6 +44,8 @@ uses type + EBuildTokenListWarning=Exception; + TBuildTokenListFlag=(btlOnlyDirectives); TBuildTokenListFlags = set of TBuildTokenListFlag; @@ -114,7 +116,7 @@ implementation uses { local } - JcfStringUtils, JcfRegistrySettings, ParseError, jcfbaseConsts; + JcfStringUtils, JcfRegistrySettings, ParseError, jcfbaseConsts, FormatFlags; const CurlyLeft = '{'; //widechar(123); @@ -543,6 +545,7 @@ begin end; end; + { complexities like 'Hello'#32'World' and #$12'Foo' are assemlbed in the parser } function TBuildTokenList.TryLiteralString(const pcToken: TSourceToken; const pcDelimiter: Char): boolean; @@ -1023,9 +1026,16 @@ var lcNew: TSourceToken; liCounter: integer; lbIncludeToken: boolean; + liParseDisabledCount: integer; + liParseDisabledCountRaw: integer; // needed to emit warning. + lsError: string; + leFlags: TFormatFlags; + lbOn: boolean; begin Assert(SourceCode <> ''); liCounter := 0; + liParseDisabledCount := 0; + liParseDisabledCountRaw := 0; while not EndOfFile do begin @@ -1035,6 +1045,31 @@ begin lcNew := GetNextToken; if lcNew<>nil then begin + if (lcNew.TokenType=ttComment) then + begin + if ReadCommentJcfFlags(lcNew.SourceCode, lsError, leFlags, lbOn) then + begin + if eParse in leFlags then + begin + if not lbOn then // //jcf:parse=off + begin + Inc(liParseDisabledCount); + Inc(liParseDisabledCountRaw); + end + else + begin + if (liParseDisabledCount > 0) then // //jcf:parse=on + Dec(liParseDisabledCount); + Dec(liParseDisabledCountRaw); + end; + end; + end; + end + else + begin + if (liParseDisabledCount > 0) and lcNew.IsSolid then + lcNew.TokenType := ttComment; + end; if btlOnlyDirectives in AFlags then begin if not ((lcNew.TokenType=ttComment) and (lcNew.CommentStyle=eCompilerDirective)) then @@ -1053,6 +1088,8 @@ begin raise; end; end; + if liParseDisabledCountRaw <> 0 then + raise EBuildTokenListWarning.Create(lisMsgImbalancedParseDisable); end; function TBuildTokenList.Current: Char; diff --git a/components/jcf2/Process/FormatFlags.pas b/components/jcf2/Process/FormatFlags.pas index 1baa4a6107..23b57432e5 100644 --- a/components/jcf2/Process/FormatFlags.pas +++ b/components/jcf2/Process/FormatFlags.pas @@ -39,7 +39,9 @@ uses type - TFormatFlag = (eAllFormat, + TFormatFlag = ( + eParse, //must be the first item. + eAllFormat, //must be the second item. eObfuscate, eAddSpace, eRemoveSpace, eAddReturn, eRemoveReturn, @@ -52,7 +54,8 @@ type { these flags control: - AllFormat: all clarify processes - turn the formatter as a whole on or off + parse: stop parsing tokens - to turn it on again need jcf:parse=on + AllFormat: all clarify processes - turn the formatter as a whole on or off (except parse) space: all processes that insert or remove spaces indent: inenting of code blocks etc return: all processes that insert or remove returns - note tat there is some overlap with @@ -72,7 +75,7 @@ const FORMAT_COMMENT_PREFIX = '//jcf:'; FORMAT_COMMENT_PREFIX_LEN = 6; - ALL_FLAGS: TFormatFlags = [Low(TFormatFlag)..High(TFormatFlag)]; + ALL_FLAGS: TFormatFlags = [eAllFormat..High(TFormatFlag)]; implementation @@ -87,8 +90,9 @@ type end; const - FORMAT_FLAG_NAMES: array[1..33] of TRFlagNameData = + FORMAT_FLAG_NAMES: array[1..34] of TRFlagNameData = ( + (sName: 'parse'; eFlags: [eParse]), (sName: 'format'; eFlags: [eAllFormat]), (sName: 'obfuscate'; eFlags: [eObfuscate]), @@ -138,6 +142,7 @@ const (sName: 'findreplaceuses'; eFlags: [eFindReplaceUses]), (sName: 'removecomments'; eFlags: [eRemoveComments]) + ); @@ -153,9 +158,9 @@ const { like StrToBoolean, but recognises 'on' and 'off' too } function LStrToBoolean(const ps: string): boolean; begin - if AnsiSameText(ps, 'on') then + if ps = 'on' then Result := True - else if AnsiSameText(ps, 'off') then + else if ps = 'off' then Result := False else Result := StrToBoolean(ps); @@ -172,7 +177,7 @@ end; function ReadCommentJcfFlags(psComment: string; out psError: string; out peFlags: TFormatFlags; out pbOn: boolean): boolean; var - lsPrefix, lsRest: string; + lsRest: string; lsSetting, lsState: string; lbFlagFound: boolean; liLoop: integer; @@ -186,10 +191,15 @@ begin else if psComment = OLD_NOFORMAT_OFF then psComment := NOFORMAT_OFF; + if length(psComment) <= FORMAT_COMMENT_PREFIX_LEN then + exit; + if (psComment[3] <> 'j') and (psComment[3] <> 'J') then + exit; + psComment := LowerCase(psComment); + { all comments without the required prefix are of no import to this code if it's not one, then exit without error } - lsPrefix := StrLeft(psComment, 6); - if not (AnsiSameText(lsPrefix, FORMAT_COMMENT_PREFIX)) then + if not CompareMem(@psComment[1], @FORMAT_COMMENT_PREFIX[1], FORMAT_COMMENT_PREFIX_LEN) then exit; // should be a valid jcf flag directive after here @@ -225,7 +235,7 @@ begin lbFlagFound := False; // accept jcf:all=on to reset state to normal by removing all flags - if AnsiSameText(lsSetting, 'all') then + if lsSetting = 'all' then begin peFlags := ALL_FLAGS; lbFlagFound := True; @@ -235,7 +245,7 @@ begin { match the setting from the table } for liLoop := low(FORMAT_FLAG_NAMES) to high(FORMAT_FLAG_NAMES) do begin - if AnsiSameText(lsSetting, FORMAT_FLAG_NAMES[liLoop].sName) then + if lsSetting = FORMAT_FLAG_NAMES[liLoop].sName then begin peFlags := FORMAT_FLAG_NAMES[liLoop].eFlags; lbFlagFound := True; diff --git a/components/jcf2/Process/Obfuscate/RemoveComment.pas b/components/jcf2/Process/Obfuscate/RemoveComment.pas index eec4ccc2de..79e1b99e52 100644 --- a/components/jcf2/Process/Obfuscate/RemoveComment.pas +++ b/components/jcf2/Process/Obfuscate/RemoveComment.pas @@ -48,7 +48,7 @@ type implementation uses - JcfStringUtils, + SysUtils, JcfStringUtils, SourceToken, Tokens, ParseTreeNodeType, FormatFlags, Converter; function CommentMustStay(const pc: TSourceToken): boolean; @@ -73,8 +73,8 @@ begin Result := True; // these are also flags - if ((pc.CommentStyle = eDoubleSlash) and - (StrLeft(pc.SourceCode, FORMAT_COMMENT_PREFIX_LEN) = FORMAT_COMMENT_PREFIX)) then + if (pc.CommentStyle = eDoubleSlash) and + (CompareText(StrLeft(pc.SourceCode, FORMAT_COMMENT_PREFIX_LEN), FORMAT_COMMENT_PREFIX) = 0) then Result := True; // these comments are used to process only selected text. diff --git a/components/jcf2/Process/SwitchableVisitor.pas b/components/jcf2/Process/SwitchableVisitor.pas index aa49ff9041..7e5db53f38 100644 --- a/components/jcf2/Process/SwitchableVisitor.pas +++ b/components/jcf2/Process/SwitchableVisitor.pas @@ -44,6 +44,8 @@ type fbEnabled: boolean; // on/off flags that this processor responds to feFormatFlags: TFormatFlags; + fiParseDisableCount: integer; + fbSpecialCommentEnabled: boolean; protected // enabled state may be changed by this token @@ -71,6 +73,8 @@ constructor TSwitchableVisitor.Create; begin inherited; fbEnabled := True; + fiParseDisableCount := 0; + fbSpecialCommentEnabled := False; //by default, format unless all processors are turned off feFormatFlags := [eAllFormat]; @@ -97,6 +101,15 @@ begin if lsError <> '' then raise TEParseError.Create(lsError, lcToken); + fbSpecialCommentEnabled := fbEnabled and (fiParseDisableCount = 0) and (not lbOn); // keep formating //jcf:XXX=off + if eParse in leFlags then + begin + if not lbOn then + Inc(fiParseDisableCount) + else if fiParseDisableCount > 0 then + Dec(fiParseDisableCount); + end; + // does this flag affect us? if (FormatFlags * leFlags) <> [] then fbEnabled := lbOn; @@ -120,10 +133,11 @@ begin InspectSourceToken(pcToken); - if fbEnabled then + if (fbEnabled and (fiParseDisableCount = 0)) or fbSpecialCommentEnabled then Result := EnabledVisitSourceToken(pcToken) else Result:= False; + fbSpecialCommentEnabled := False; end; end. diff --git a/components/jcf2/ReadWrite/Converter.pas b/components/jcf2/ReadWrite/Converter.pas index f484572fb6..762b18233a 100644 --- a/components/jcf2/ReadWrite/Converter.pas +++ b/components/jcf2/ReadWrite/Converter.pas @@ -171,6 +171,10 @@ begin try fcTokeniser.BuildTokenList(lcTokenList); except + on E: EBuildTokenListWarning do + begin + SendStatusMessage('', Format(lisMsgWarningClassMsg, ['', E.Message]), mtCodeWarning, -1, -1); + end; on E: Exception do begin fbConvertError := True; diff --git a/components/jcf2/docs/special_comments.md b/components/jcf2/docs/special_comments.md index 0046db35a9..617bd85f89 100644 --- a/components/jcf2/docs/special_comments.md +++ b/components/jcf2/docs/special_comments.md @@ -1,5 +1,3 @@ - JCF comments - Special comments in the Jedi Code Formatter =========================================== @@ -11,20 +9,22 @@ From version 0.52, a more extensive syntax has been implemented to allow for fin For instance, in the following line of code, the formatter will not alter the spacing -`//jcf:space=off -  a   :=     a     +     1   ; -//jcf:space=on` +``` +//jcf:space=off +  a   :=     a     +     1   ; +//jcf:space=on +``` As you can see, the new syntax has the form of a comment like `//jcf:**flag**=**state**`, where **state** is one of `on` or `off`, and the flags are listed below. Flags ----- -| | | -| --- | --- | | **Flag** | **Description** | -| `all` | all=on is used to reverse the effects of all `off` comments, as described below. `all=off` is not valid | -| `format` | All formatting | +| --- | --- | +| `parse` | Stop parsing tokens (to avoid errors on unsupported code)- to turn it on again need jcf:parse=on (since lazarus 3.99)| +| `all` | all=on is used to reverse the effects of all `off` comments (except parse), as described below. `all=off` is not valid | +| `format` | All formatting (except parse)| | `space` | All processes that insert or remove spaces | | `addspace` | All processes that insert spaces | | `removespace` | All processes that remove spaces | @@ -56,9 +56,9 @@ Flags The following rules apply: -* Turning a flag on will not enable it if it is not disabled in the configuration - the flags are to 'block out' a section of code where a format option is not wanted. The on state is there only to end a previous off state. -* All processes turned off buy a flag comment remain turned off until the corresponding flag comment that turns it back on is reached, or the end of the file is reached, or the flag comment `//jcf:all=on` is reached. -* The flag comment `//jcf:all=on` will reenable all processes that have been turned off by comments, ie it returns formatting to the options specified in the configuration. +* Turning a flag on will not enable it if it is not enabled in the configuration - the flags are to 'block out' a section of code where a format option is not wanted. The on state is there only to end a previous off state. +* All processes (except parse) turned off by a flag comment remain turned off until the corresponding flag comment that turns it back on is reached, or the end of the file is reached, or the flag comment `//jcf:all=on` is reached. +* The flag comment `//jcf:all=on` will reenable all processes that have been turned off by comments, ie it returns formatting to the options specified in the configuration (except parse). * Flag comments only affect the file in which they occur, from the position in which the occur onwards. * The effects of some of the flags overlap - this is done to allow degrees of granularity e.g. turn off all alignment, or just alignment of variable declarations for a block of code. * In these special comments, character case and some spaces are ignored. @@ -68,4 +68,3 @@ For examples see the test case unit TestExclusionFlags.pas, for implmentation se * * * -[Index](index.html) \ No newline at end of file diff --git a/components/jcf2/jcfbaseconsts.pas b/components/jcf2/jcfbaseconsts.pas index 7a9520cec0..66a55c77e6 100644 --- a/components/jcf2/jcfbaseconsts.pas +++ b/components/jcf2/jcfbaseconsts.pas @@ -37,6 +37,8 @@ const lisMsgEmptyFinallyEndBlock = 'Empty finally..end block'; lisMsgEmptyTryBlock = 'Empty try block'; lisMsgExceptionClassMsg = 'Exception %s %s'; + lisMsgWarningClassMsg = 'Warning %s %s'; + lisMsgImbalancedParseDisable = 'Imbalanced number of: //jcf:parse=off/on'; lisMsgExceptionParsing = 'Exception parsing "%s": %s'; lisMsgExceptionTokenising = 'Exception tokenising "%s": %s'; lisMsgExistsAlreadyRemoveIt = '%s %s %s exists already. Remove it?';