From 69edc4d8c0a7f91cf77a55091aef48ced4e92f5f Mon Sep 17 00:00:00 2001 From: mattias Date: Fri, 13 Feb 2015 20:34:21 +0000 Subject: [PATCH] lazutils: moved ReplaceSubString to LazUTF8, added UTF8QuotedStr git-svn-id: trunk@47748 - --- components/codetools/fileprocs.pas | 4 +- components/lazutils/lazlogger.pas | 50 +---------- components/lazutils/lazutf8.pas | 101 ++++++++++++++++++--- components/wiki/lazwiki/wikiparser.pas | 2 +- components/wiki/wikiget.lpr | 2 +- docs/ExtendingTheIDE.txt | 120 +------------------------ docs/xml/lazutils/lazutf8.xml | 6 +- ide/etquickfixes.pas | 3 +- ide/modematrixopts.pas | 2 +- test/lazutils/testlazutf8.pas | 20 +++++ test/lazutils/testlazutils.pas | 2 +- 11 files changed, 125 insertions(+), 187 deletions(-) diff --git a/components/codetools/fileprocs.pas b/components/codetools/fileprocs.pas index ceff37bce5..471b039b88 100644 --- a/components/codetools/fileprocs.pas +++ b/components/codetools/fileprocs.pas @@ -39,8 +39,8 @@ uses {$IFDEF Windows} Windows, {$ENDIF} - Classes, SysUtils, LazUtilities, LazUTF8, LazDbgLog, LazFileCache, - LazFileUtils, LazUTF8Classes, LazLogger, AVL_Tree, CodeToolsStrConsts; + Classes, SysUtils, LazUtilities, LazDbgLog, LazLogger, LazUTF8, LazFileCache, + LazFileUtils, LazUTF8Classes, AVL_Tree, CodeToolsStrConsts; type TFPCStreamSeekType = int64; diff --git a/components/lazutils/lazlogger.pas b/components/lazutils/lazlogger.pas index f014c891e4..85ba6915eb 100644 --- a/components/lazutils/lazlogger.pas +++ b/components/lazutils/lazlogger.pas @@ -4,7 +4,7 @@ unit LazLogger; interface uses - Classes, SysUtils, types, math, LazLoggerBase, LazClasses, FileUtil; + Classes, SysUtils, types, math, LazUTF8, LazLoggerBase, LazClasses, FileUtil; type @@ -21,7 +21,7 @@ function DbgWideStr(const StringWithSpecialChars: widestring): string; overload; function ConvertLineEndings(const s: string): string; procedure ReplaceSubstring(var s: string; StartPos, Count: SizeInt; - const Insertion: string); + const Insertion: string); inline; deprecated; type @@ -785,52 +785,8 @@ end; procedure ReplaceSubstring(var s: string; StartPos, Count: SizeInt; const Insertion: string); -var - MaxCount: SizeInt; - InsertionLen: SizeInt; - SLen: SizeInt; - RestLen: SizeInt; - p: PByte; begin - SLen:=length(s); - if StartPos>SLen then begin - s:=s+Insertion; - exit; - end; - if StartPos<1 then StartPos:=1; - if Count<0 then Count:=0; - MaxCount:=SLen-StartPos+1; - if Count>MaxCount then - Count:=MaxCount; - InsertionLen:=length(Insertion); - if (Count=0) and (InsertionLen=0) then - exit; // nothing to do - if (Count=InsertionLen) then begin - if CompareMem(PByte(s)+StartPos-1,Pointer(Insertion),Count) then - // already the same content - exit; - UniqueString(s); - end else begin - RestLen:=SLen-StartPos-Count+1; - if InsertionLen0 then begin - UniqueString(s); - p:=PByte(s)+StartPos-1; - System.Move((p+Count)^,(p+InsertionLen)^,RestLen); - end; - Setlength(s,SLen-Count+InsertionLen); - end else begin - // longen - Setlength(s,SLen-Count+InsertionLen); - if RestLen>0 then begin - p:=PByte(s)+StartPos-1; - System.Move((p+Count)^,(p+InsertionLen)^,RestLen); - end; - end; - end; - if InsertionLen>0 then - System.Move(PByte(Insertion)^,(PByte(s)+StartPos-1)^,InsertionLen); + LazUTF8.ReplaceSubstring(s,StartPos,Count,Insertion); end; initialization diff --git a/components/lazutils/lazutf8.pas b/components/lazutils/lazutf8.pas index ac51d54b73..60eb3e1b49 100644 --- a/components/lazutils/lazutf8.pas +++ b/components/lazutils/lazutf8.pas @@ -95,16 +95,17 @@ function UTF8LowerString(const s: string): string; function UTF8UpperCase(const AInStr: string; ALanguage: string=''): string; function UTF8UpperString(const s: string): string; function FindInvalidUTF8Character(p: PChar; Count: PtrInt; - StopOnNonASCII: Boolean = true): PtrInt; + StopOnNonUTF8: Boolean = true): PtrInt; function ValidUTF8String(const s: String): String; -function Utf8StringOfChar(AUtf8Char: String; N: Integer): String; -function Utf8AddChar(AUtf8Char: String; const S: String; N: Integer): String; -function Utf8AddCharR(AUtf8Char: String; const S: String; N: Integer): String; +function UTF8StringOfChar(AUtf8Char: String; N: Integer): String; +function UTF8AddChar(AUtf8Char: String; const S: String; N: Integer): String; +function UTF8AddCharR(AUtf8Char: String; const S: String; N: Integer): String; function UTF8PadLeft(const S: String; const N: Integer; const AUtf8Char: String = #32): String; function UTF8PadRight(const S: String; const N: Integer; const AUtf8Char: String = #32): String; function UTF8PadCenter(const S: String; const N: Integer; const AUtf8Char: String = #32): String; -function Utf8LeftStr(const AText: String; const ACount: Integer): String; -function Utf8RightStr(const AText: String; const ACount: Integer): String; +function UTF8LeftStr(const AText: String; const ACount: Integer): String; +function UTF8RightStr(const AText: String; const ACount: Integer): String; +function UTF8QuotedStr(const S, Quote: string): string; //Utf8 version of MidStr is just Utf8Copy with same parameters, so it is not implemented here type @@ -158,6 +159,9 @@ procedure LazGetShortLanguageID(var Lang: String); var FPUpChars: array[char] of char; +procedure ReplaceSubstring(var s: string; StartPos, Count: SizeInt; + const Insertion: string); + implementation uses @@ -2482,7 +2486,7 @@ end; function FindInvalidUTF8Character(p: PChar; Count: PtrInt; - StopOnNonASCII: Boolean): PtrInt; + StopOnNonUTF8: Boolean): PtrInt; // return -1 if ok var CharLen: Integer; @@ -2493,12 +2497,12 @@ begin while Result=192) then + if StopOnNonUTF8 or (ord(c)>=192) then exit; CharLen:=1; end else if ord(c)<=%11011111 then begin @@ -2533,7 +2537,7 @@ begin exit; // missing following bytes end else begin - if StopOnNonASCII then + if StopOnNonUTF8 then exit; CharLen:=1; end; @@ -2698,8 +2702,31 @@ begin Result := Utf8Copy(AText,l-j+1,j); end; - - +function UTF8QuotedStr(const S, Quote: string): string; +// replace all Quote in S with double Quote and enclose the result in Quote. +var + QuoteC: Char; + p, QuoteP, CopyPos: PChar; + QuoteLen: SizeInt; +begin + Result:=Quote; + p:=PChar(S); + CopyPos:=p; + QuoteC:=Quote[1]; + QuoteP:=PChar(Quote); + QuoteLen:=length(Quote); + repeat + if (p^=#0) and (p-PChar(S)=length(S)) then + break; + if (p^=QuoteC) and CompareMem(p,QuoteP,QuoteLen) then begin + inc(p,QuoteLen); + Result+=copy(S,CopyPos-PChar(S)+1,p-CopyPos)+Quote; + CopyPos:=p; + end else + inc(p); + until false; + Result+=copy(S,CopyPos-PChar(S)+1,p-CopyPos)+Quote; +end; function UTF8Trim(const s: string; Flags: TUTF8TrimFlags): string; var @@ -3380,6 +3407,56 @@ begin if Length(Lang) > 2 then Lang := Lang[1] + Lang[2]; end; +procedure ReplaceSubstring(var s: string; StartPos, Count: SizeInt; + const Insertion: string); +var + MaxCount: SizeInt; + InsertionLen: SizeInt; + SLen: SizeInt; + RestLen: SizeInt; + p: PByte; +begin + SLen:=length(s); + if StartPos>SLen then begin + s:=s+Insertion; + exit; + end; + if StartPos<1 then StartPos:=1; + if Count<0 then Count:=0; + MaxCount:=SLen-StartPos+1; + if Count>MaxCount then + Count:=MaxCount; + InsertionLen:=length(Insertion); + if (Count=0) and (InsertionLen=0) then + exit; // nothing to do + if (Count=InsertionLen) then begin + if CompareMem(PByte(s)+StartPos-1,Pointer(Insertion),Count) then + // already the same content + exit; + UniqueString(s); + end else begin + RestLen:=SLen-StartPos-Count+1; + if InsertionLen0 then begin + UniqueString(s); + p:=PByte(s)+StartPos-1; + System.Move((p+Count)^,(p+InsertionLen)^,RestLen); + end; + Setlength(s,SLen-Count+InsertionLen); + end else begin + // longen + Setlength(s,SLen-Count+InsertionLen); + if RestLen>0 then begin + p:=PByte(s)+StartPos-1; + System.Move((p+Count)^,(p+InsertionLen)^,RestLen); + end; + end; + end; + if InsertionLen>0 then + System.Move(PByte(Insertion)^,(PByte(s)+StartPos-1)^,InsertionLen); +end; + procedure InitFPUpchars; var c: Char; diff --git a/components/wiki/lazwiki/wikiparser.pas b/components/wiki/lazwiki/wikiparser.pas index 968e98213b..eb99431a6b 100644 --- a/components/wiki/lazwiki/wikiparser.pas +++ b/components/wiki/lazwiki/wikiparser.pas @@ -34,7 +34,7 @@ unit WikiParser; interface uses - Classes, SysUtils, laz2_XMLRead, laz2_DOM, LazUTF8, LazLogger, + Classes, SysUtils, laz2_XMLRead, laz2_DOM, LazLogger, LazUTF8, BasicCodeTools, KeywordFuncLists; const diff --git a/components/wiki/wikiget.lpr b/components/wiki/wikiget.lpr index 9272abec75..a561d653e4 100644 --- a/components/wiki/wikiget.lpr +++ b/components/wiki/wikiget.lpr @@ -30,7 +30,7 @@ uses cthreads, {$ENDIF} Classes, SysUtils, LazFileUtils, laz2_XMLRead, laz2_DOM, laz2_XMLWrite, - LazUTF8, LazLogger, CodeToolsStructs, CustApp, AVL_Tree, + LazLogger, LazUTF8, CodeToolsStructs, CustApp, AVL_Tree, {$IF FPC_FULLVERSION<20701} myfphttpclient, {$ELSE} diff --git a/docs/ExtendingTheIDE.txt b/docs/ExtendingTheIDE.txt index 3106433d10..f01c49fae9 100644 --- a/docs/ExtendingTheIDE.txt +++ b/docs/ExtendingTheIDE.txt @@ -1,125 +1,7 @@ Extending the IDE (Overview) ============================ -The online pages are more up to date and have more examples: +See the online pages: http://wiki.lazarus.freepascal.org/Extending_the_IDE - -The IDE supports several types of plugins: - -Components - These are the items in the component palette. For a example TButton can be - used to create Buttons. - -Component Editors - Component editors are used when you double click on a component in the - designer or to add some extra items to the popup menu of the designer, when - you right click on a component. - -Property Editors - These are used by the rows in the object inspector. - -Experts - These are all other types. - - -------------------------------------------------------------------------------- -There are two possibilities to add your own plugins to Lazarus: - -1. Write a package, install it and register your plugins in the 'Register' - procedure of a unit. -2. Extend the lazarus code, and send your cvs diff to the lazarus mailing list. - - -------------------------------------------------------------------------------- -Writing components: - -ToDo -Hint: Create a new component via the package editor. - - - -------------------------------------------------------------------------------- -Writing component editors: - -ToDo -Hint: see componenteditors.pas for examples - - -------------------------------------------------------------------------------- -Writing property editors - -ToDo -Hint: see propedits.pp for examples - - -------------------------------------------------------------------------------- -Register event handlers - -There are several events in the IDE, for which plugins can add their own -handlers. -In propedits.pp there is a GlobalDesignHook object, which maintains several -events for designing. Each event calls a list of handlers. The default handlers -are added by the IDE. You can add your own handlers with the AddHandlerXXX and -RemoveHandlerXXX methods. They will be called before the default handlers. -Examples: - - Adding your handler (this is normally done in the constructor of your object): - GlobalDesignHook.AddHandlerComponentAdded(@YourOnComponentAdded); - - Removing your handler: - GlobalDesignHook.RemoveHandlerComponentAdded(@YourOnComponentAdded); - - You can remove all handlers at once. For example, it is a good idea to add - this line in the destructor of object: - GlobalDesignHook.RemoveAllHandlersForObject(Self); - - -The handlers of GlobalDesignHook: - - // lookup root - ChangeLookupRoot - Called when the LookupRoot changed. - The LookupRoot is the owner object of the currently selected components. - Normally this is a TForm. - - // methods - CreateMethod - GetMethodName - GetMethods - MethodExists - RenameMethod - ShowMethod - Called - MethodFromAncestor - ChainCall - - // components - GetComponent - GetComponentName - GetComponentNames - GetRootClassName - ComponentRenamed - Called when a component was renamed - ComponentAdded - Called when a new component was added to the LookupRoot - ComponentDeleting - Called before a component is freed. - DeleteComponent - Called by the IDE to delete a component. - GetSelectedComponents - Get the current selection of components. - - // persistent objects - GetObject - GetObjectName - GetObjectNames - - // modifing - Modified - Revert - RefreshPropertyValues - - - diff --git a/docs/xml/lazutils/lazutf8.xml b/docs/xml/lazutils/lazutf8.xml index b55d46e69e..c8ffb19ee2 100644 --- a/docs/xml/lazutils/lazutf8.xml +++ b/docs/xml/lazutils/lazutf8.xml @@ -444,8 +444,10 @@ Returns 0 if not found. - - + Returns -1 if ok, otherwise byte index of invalid UTF8 codepoint + It always stops on irregular codepoints. For example Codepoint 0 is normally encoded as #0, but it can also be encoded as #192#0. Because most software does not check this, it can be exploited and is a security risk. + +If StopOnNonUTF8 is false it will ignore undefined codes. For example #128. By default it stops on such codes. diff --git a/ide/etquickfixes.pas b/ide/etquickfixes.pas index 8889f73c4f..3d93c3a9ba 100644 --- a/ide/etquickfixes.pas +++ b/ide/etquickfixes.pas @@ -24,6 +24,7 @@ Standard Quick Fixes - tools to help fixing (compiler) messages. ToDo: + - cant find unit: duplicate include file, e.g. control.inc - TQuickFixIdentifierNotFoundAddLocal: extend with add private/public - local var not used: remove declaration and all assignments - There is no method in an ancestor class to be overriden: @@ -53,7 +54,7 @@ interface uses Classes, SysUtils, - LazLogger, AvgLvlTree, LazFileUtils, + LazLogger, AvgLvlTree, LazFileUtils, LazUTF8, Menus, Dialogs, Controls, CodeToolManager, CodeCache, CodeTree, CodeAtom, BasicCodeTools, KeywordFuncLists, diff --git a/ide/modematrixopts.pas b/ide/modematrixopts.pas index 88e88847c0..641b03dc0d 100644 --- a/ide/modematrixopts.pas +++ b/ide/modematrixopts.pas @@ -26,7 +26,7 @@ unit ModeMatrixOpts; interface uses - Classes, SysUtils, contnrs, LazConfigStorage, Laz2_XMLCfg, LazLogger, + Classes, SysUtils, contnrs, LazConfigStorage, Laz2_XMLCfg, LazLogger, LazUTF8, FileProcs, KeywordFuncLists, CodeToolsCfgScript, LazarusIDEStrConsts; const diff --git a/test/lazutils/testlazutf8.pas b/test/lazutils/testlazutf8.pas index 47732466a4..ae76d102b9 100644 --- a/test/lazutils/testlazutf8.pas +++ b/test/lazutils/testlazutf8.pas @@ -6,6 +6,7 @@ ./runtests --format=plain --suite=TestUTF8Trim ./runtests --format=plain --suite=TestUTF8Pos ./runtests --format=plain --suite=TestFindInvalidUTF8 + ./runtests --format=plain --suite=TestUTF8QuotedStr } unit TestLazUTF8; @@ -26,6 +27,7 @@ type procedure TestUTF8Trim; procedure TestUTF8Pos; procedure TestFindInvalidUTF8; + procedure TestUTF8QuotedStr; end; implementation @@ -103,6 +105,24 @@ begin t(#$f0#$8f#$bf#$bf,0,'invalid: $ffff encoded as 4 byte'); end; +procedure TTestLazUTF8.TestUTF8QuotedStr; + + procedure t(const S, Quote, Expected: string); + var + Actual: String; + begin + Actual:=UTF8QuotedStr(S,Quote); + AssertEquals('S="'+S+'" Quote="'+Quote+'"',Expected,Actual); + end; + +begin + t('','=','=='); + t('','AB','ABAB'); + t('A','A','AAAA'); + t('bAb','A','AbAAbA'); + t('cABc','AB','ABcABABcAB'); +end; + initialization AddToLazUtilsTestSuite(TTestLazUTF8); diff --git a/test/lazutils/testlazutils.pas b/test/lazutils/testlazutils.pas index 2771cd07c0..3df851fdb1 100644 --- a/test/lazutils/testlazutils.pas +++ b/test/lazutils/testlazutils.pas @@ -15,7 +15,7 @@ unit TestLazUtils; interface uses - Classes, SysUtils, fpcunit, testglobals, LazLogger, LazFileUtils; + Classes, SysUtils, fpcunit, testglobals, LazLogger, LazUTF8, LazFileUtils; type