lazutils: moved ReplaceSubString to LazUTF8, added UTF8QuotedStr

git-svn-id: trunk@47748 -

components/codetools/fileprocs.pas

@@ -39,8 +39,8 @@ uses
   {$IFDEF Windows}
   Windows,
   {$ENDIF}
-  Classes, SysUtils, LazUtilities, LazUTF8, LazDbgLog, LazFileCache,
-  LazFileUtils, LazUTF8Classes, LazLogger, AVL_Tree, CodeToolsStrConsts;
+  Classes, SysUtils, LazUtilities, LazDbgLog, LazLogger, LazUTF8, LazFileCache,
+  LazFileUtils, LazUTF8Classes, AVL_Tree, CodeToolsStrConsts;
 
 type
   TFPCStreamSeekType = int64;

components/lazutils/lazlogger.pas

@@ -4,7 +4,7 @@ unit LazLogger;
 interface
 
 uses
-  Classes, SysUtils, types, math, LazLoggerBase, LazClasses, FileUtil;
+  Classes, SysUtils, types, math, LazUTF8, LazLoggerBase, LazClasses, FileUtil;
 
 type
 
@@ -21,7 +21,7 @@ function DbgWideStr(const StringWithSpecialChars: widestring): string; overload;
 
 function ConvertLineEndings(const s: string): string;
 procedure ReplaceSubstring(var s: string; StartPos, Count: SizeInt;
-                           const Insertion: string);
+                           const Insertion: string); inline; deprecated;
 
 type
 
@@ -785,52 +785,8 @@ end;
 
 procedure ReplaceSubstring(var s: string; StartPos, Count: SizeInt;
   const Insertion: string);
-var
-  MaxCount: SizeInt;
-  InsertionLen: SizeInt;
-  SLen: SizeInt;
-  RestLen: SizeInt;
-  p: PByte;
 begin
-  SLen:=length(s);
-  if StartPos>SLen then begin
-    s:=s+Insertion;
-    exit;
-  end;
-  if StartPos<1 then StartPos:=1;
-  if Count<0 then Count:=0;
-  MaxCount:=SLen-StartPos+1;
-  if Count>MaxCount then
-    Count:=MaxCount;
-  InsertionLen:=length(Insertion);
-  if (Count=0) and (InsertionLen=0) then
-    exit; // nothing to do
-  if (Count=InsertionLen) then begin
-    if CompareMem(PByte(s)+StartPos-1,Pointer(Insertion),Count) then
-      // already the same content
-      exit;
-    UniqueString(s);
-  end else begin
-    RestLen:=SLen-StartPos-Count+1;
-    if InsertionLen<Count then begin
-      // shorten
-      if RestLen>0 then begin
-        UniqueString(s);
-        p:=PByte(s)+StartPos-1;
-        System.Move((p+Count)^,(p+InsertionLen)^,RestLen);
-      end;
-      Setlength(s,SLen-Count+InsertionLen);
-    end else begin
-      // longen
-      Setlength(s,SLen-Count+InsertionLen);
-      if RestLen>0 then begin
-        p:=PByte(s)+StartPos-1;
-        System.Move((p+Count)^,(p+InsertionLen)^,RestLen);
-      end;
-    end;
-  end;
-  if InsertionLen>0 then
-    System.Move(PByte(Insertion)^,(PByte(s)+StartPos-1)^,InsertionLen);
+  LazUTF8.ReplaceSubstring(s,StartPos,Count,Insertion);
 end;
 
 initialization

components/lazutils/lazutf8.pas

@@ -95,16 +95,17 @@ function UTF8LowerString(const s: string): string;
 function UTF8UpperCase(const AInStr: string; ALanguage: string=''): string;
 function UTF8UpperString(const s: string): string;
 function FindInvalidUTF8Character(p: PChar; Count: PtrInt;
-                                  StopOnNonASCII: Boolean = true): PtrInt;
+                                  StopOnNonUTF8: Boolean = true): PtrInt;
 function ValidUTF8String(const s: String): String;
-function Utf8StringOfChar(AUtf8Char: String; N: Integer): String;
-function Utf8AddChar(AUtf8Char: String; const S: String; N: Integer): String;
-function Utf8AddCharR(AUtf8Char: String; const S: String; N: Integer): String;
+function UTF8StringOfChar(AUtf8Char: String; N: Integer): String;
+function UTF8AddChar(AUtf8Char: String; const S: String; N: Integer): String;
+function UTF8AddCharR(AUtf8Char: String; const S: String; N: Integer): String;
 function UTF8PadLeft(const S: String; const N: Integer; const AUtf8Char: String = #32): String;
 function UTF8PadRight(const S: String; const N: Integer; const AUtf8Char: String = #32): String;
 function UTF8PadCenter(const S: String; const N: Integer; const AUtf8Char: String = #32): String;
-function Utf8LeftStr(const AText: String; const ACount: Integer): String;
-function Utf8RightStr(const AText: String; const ACount: Integer): String;
+function UTF8LeftStr(const AText: String; const ACount: Integer): String;
+function UTF8RightStr(const AText: String; const ACount: Integer): String;
+function UTF8QuotedStr(const S, Quote: string): string;
 //Utf8 version of MidStr is just Utf8Copy with same parameters, so it is not implemented here
 
 type
@@ -158,6 +159,9 @@ procedure LazGetShortLanguageID(var Lang: String);
 var
   FPUpChars: array[char] of char;
 
+procedure ReplaceSubstring(var s: string; StartPos, Count: SizeInt;
+                           const Insertion: string);
+
 implementation
 
 uses
@@ -2482,7 +2486,7 @@ end;
 
 
 function FindInvalidUTF8Character(p: PChar; Count: PtrInt;
-  StopOnNonASCII: Boolean): PtrInt;
+  StopOnNonUTF8: Boolean): PtrInt;
 // return -1 if ok
 var
   CharLen: Integer;
@@ -2493,12 +2497,12 @@ begin
     while Result<Count do begin
       c:=p^;
       if ord(c)<%10000000 then begin
-        // regular single byte ASCII character (#0 is a character, this is pascal ;)
+        // regular single byte ASCII character (#0 is a character, this is Pascal ;)
         CharLen:=1;
       end else if ord(c)<=%11000001 then begin
         // single byte character, between valid UTF-8 encodings
         // %11000000 and %11000001 map 2 byte to #0..#128, which is invalid and used for XSS attacks
-        if StopOnNonASCII or (ord(c)>=192) then
+        if StopOnNonUTF8 or (ord(c)>=192) then
           exit;
         CharLen:=1;
       end else if ord(c)<=%11011111 then begin
@@ -2533,7 +2537,7 @@ begin
           exit; // missing following bytes
       end
       else begin
-        if StopOnNonASCII then
+        if StopOnNonUTF8 then
           exit;
         CharLen:=1;
       end;
@@ -2698,8 +2702,31 @@ begin
   Result := Utf8Copy(AText,l-j+1,j);
 end;
 
-
-
+function UTF8QuotedStr(const S, Quote: string): string;
+// replace all Quote in S with double Quote and enclose the result in Quote.
+var
+  QuoteC: Char;
+  p, QuoteP, CopyPos: PChar;
+  QuoteLen: SizeInt;
+begin
+  Result:=Quote;
+  p:=PChar(S);
+  CopyPos:=p;
+  QuoteC:=Quote[1];
+  QuoteP:=PChar(Quote);
+  QuoteLen:=length(Quote);
+  repeat
+    if (p^=#0) and (p-PChar(S)=length(S)) then
+      break;
+    if (p^=QuoteC) and CompareMem(p,QuoteP,QuoteLen) then begin
+      inc(p,QuoteLen);
+      Result+=copy(S,CopyPos-PChar(S)+1,p-CopyPos)+Quote;
+      CopyPos:=p;
+    end else
+      inc(p);
+  until false;
+  Result+=copy(S,CopyPos-PChar(S)+1,p-CopyPos)+Quote;
+end;
 
 function UTF8Trim(const s: string; Flags: TUTF8TrimFlags): string;
 var
@@ -3380,6 +3407,56 @@ begin
   if Length(Lang) > 2 then Lang := Lang[1] + Lang[2];
 end;
 
+procedure ReplaceSubstring(var s: string; StartPos, Count: SizeInt;
+  const Insertion: string);
+var
+  MaxCount: SizeInt;
+  InsertionLen: SizeInt;
+  SLen: SizeInt;
+  RestLen: SizeInt;
+  p: PByte;
+begin
+  SLen:=length(s);
+  if StartPos>SLen then begin
+    s:=s+Insertion;
+    exit;
+  end;
+  if StartPos<1 then StartPos:=1;
+  if Count<0 then Count:=0;
+  MaxCount:=SLen-StartPos+1;
+  if Count>MaxCount then
+    Count:=MaxCount;
+  InsertionLen:=length(Insertion);
+  if (Count=0) and (InsertionLen=0) then
+    exit; // nothing to do
+  if (Count=InsertionLen) then begin
+    if CompareMem(PByte(s)+StartPos-1,Pointer(Insertion),Count) then
+      // already the same content
+      exit;
+    UniqueString(s);
+  end else begin
+    RestLen:=SLen-StartPos-Count+1;
+    if InsertionLen<Count then begin
+      // shorten
+      if RestLen>0 then begin
+        UniqueString(s);
+        p:=PByte(s)+StartPos-1;
+        System.Move((p+Count)^,(p+InsertionLen)^,RestLen);
+      end;
+      Setlength(s,SLen-Count+InsertionLen);
+    end else begin
+      // longen
+      Setlength(s,SLen-Count+InsertionLen);
+      if RestLen>0 then begin
+        p:=PByte(s)+StartPos-1;
+        System.Move((p+Count)^,(p+InsertionLen)^,RestLen);
+      end;
+    end;
+  end;
+  if InsertionLen>0 then
+    System.Move(PByte(Insertion)^,(PByte(s)+StartPos-1)^,InsertionLen);
+end;
+
 procedure InitFPUpchars;
 var
   c: Char;

components/wiki/lazwiki/wikiparser.pas

@@ -34,7 +34,7 @@ unit WikiParser;
 interface
 
 uses
-  Classes, SysUtils, laz2_XMLRead, laz2_DOM, LazUTF8, LazLogger,
+  Classes, SysUtils, laz2_XMLRead, laz2_DOM, LazLogger, LazUTF8,
   BasicCodeTools, KeywordFuncLists;
 
 const

components/wiki/wikiget.lpr

@@ -30,7 +30,7 @@ uses
   cthreads,
   {$ENDIF}
   Classes, SysUtils, LazFileUtils, laz2_XMLRead, laz2_DOM, laz2_XMLWrite,
-  LazUTF8, LazLogger, CodeToolsStructs, CustApp, AVL_Tree,
+  LazLogger, LazUTF8, CodeToolsStructs, CustApp, AVL_Tree,
   {$IF FPC_FULLVERSION<20701}
   myfphttpclient,
   {$ELSE}

docs/ExtendingTheIDE.txt

@@ -1,125 +1,7 @@
 Extending the IDE (Overview)
 ============================
 
-The online pages are more up to date and have more examples:
+See the online pages:
 http://wiki.lazarus.freepascal.org/Extending_the_IDE
 
 
-
-The IDE supports several types of plugins:
-
-Components
-  These are the items in the component palette. For a example TButton can be
-  used to create Buttons.
-    
-Component Editors
-  Component editors are used when you double click on a component in the
-  designer or to add some extra items to the popup menu of the designer, when
-  you right click on a component.
-
-Property Editors
-  These are used by the rows in the object inspector.
-
-Experts
-  These are all other types.
-
-
--------------------------------------------------------------------------------
-There are two possibilities to add your own plugins to Lazarus:
-
-1. Write a package, install it and register your plugins in the 'Register'
-   procedure of a unit.
-2. Extend the lazarus code, and send your cvs diff to the lazarus mailing list.
-
-
--------------------------------------------------------------------------------
-Writing components:
-
-ToDo
-Hint: Create a new component via the package editor.
-
-
-
--------------------------------------------------------------------------------
-Writing component editors:
-
-ToDo
-Hint: see componenteditors.pas for examples
-
-
--------------------------------------------------------------------------------
-Writing property editors
-
-ToDo
-Hint: see propedits.pp for examples
-
-
--------------------------------------------------------------------------------
-Register event handlers
-
-There are several events in the IDE, for which plugins can add their own
-handlers.
-In propedits.pp there is a GlobalDesignHook object, which maintains several
-events for designing. Each event calls a list of handlers. The default handlers
-are added by the IDE. You can add your own handlers with the AddHandlerXXX and
-RemoveHandlerXXX methods. They will be called before the default handlers.
-Examples:
-
-  Adding your handler (this is normally done in the constructor of your object):
-    GlobalDesignHook.AddHandlerComponentAdded(@YourOnComponentAdded);
-    
-  Removing your handler:
-    GlobalDesignHook.RemoveHandlerComponentAdded(@YourOnComponentAdded);
-
-  You can remove all handlers at once. For example, it is a good idea to add
-  this line in the destructor of object:
-    GlobalDesignHook.RemoveAllHandlersForObject(Self);
-
-
-The handlers of GlobalDesignHook:
-
-  // lookup root
-  ChangeLookupRoot
-    Called when the LookupRoot changed.
-    The LookupRoot is the owner object of the currently selected components.
-    Normally this is a TForm.
-
-  // methods
-  CreateMethod
-  GetMethodName
-  GetMethods
-  MethodExists
-  RenameMethod
-  ShowMethod
-    Called
-  MethodFromAncestor
-  ChainCall
-
-  // components
-  GetComponent
-  GetComponentName
-  GetComponentNames
-  GetRootClassName
-  ComponentRenamed
-    Called when a component was renamed
-  ComponentAdded
-    Called when a new component was added to the LookupRoot
-  ComponentDeleting
-    Called before a component is freed.
-  DeleteComponent
-    Called by the IDE to delete a component.
-  GetSelectedComponents
-    Get the current selection of components.
-
-  // persistent objects
-  GetObject
-  GetObjectName
-  GetObjectNames
-
-  // modifing
-  Modified
-  Revert
-  RefreshPropertyValues
-
-
-

docs/xml/lazutils/lazutf8.xml

@@ -444,8 +444,10 @@ Returns 0 if not found.
       </element>
       <!-- function Visibility: default -->
       <element name="FindInvalidUTF8Character">
-        <short/>
-        <descr/>
+        <short>Returns -1 if ok, otherwise byte index of invalid UTF8 codepoint</short>
+        <descr>It always stops on irregular codepoints. For example Codepoint 0 is normally encoded as #0, but it can also be encoded as #192#0. Because most software does not check this, it can be exploited and is a security risk.
+
+If StopOnNonUTF8 is false it will ignore undefined codes. For example #128. By default it stops on such codes.</descr>
         <errors/>
         <seealso/>
       </element>

ide/etquickfixes.pas

@@ -24,6 +24,7 @@
     Standard Quick Fixes - tools to help fixing (compiler) messages.
 
   ToDo:
+    - cant find unit: duplicate include file, e.g. control.inc
     - TQuickFixIdentifierNotFoundAddLocal: extend with add private/public
     - local var not used: remove declaration and all assignments
     - There is no method in an ancestor class to be overriden:
@@ -53,7 +54,7 @@ interface
 
 uses
   Classes, SysUtils,
-  LazLogger, AvgLvlTree, LazFileUtils,
+  LazLogger, AvgLvlTree, LazFileUtils, LazUTF8,
   Menus, Dialogs, Controls,
   CodeToolManager, CodeCache, CodeTree, CodeAtom, BasicCodeTools,
   KeywordFuncLists,

ide/modematrixopts.pas

@@ -26,7 +26,7 @@ unit ModeMatrixOpts;
 interface
 
 uses
-  Classes, SysUtils, contnrs, LazConfigStorage, Laz2_XMLCfg, LazLogger,
+  Classes, SysUtils, contnrs, LazConfigStorage, Laz2_XMLCfg, LazLogger, LazUTF8,
   FileProcs, KeywordFuncLists, CodeToolsCfgScript, LazarusIDEStrConsts;
 
 const

test/lazutils/testlazutf8.pas

@@ -6,6 +6,7 @@
      ./runtests --format=plain --suite=TestUTF8Trim
      ./runtests --format=plain --suite=TestUTF8Pos
      ./runtests --format=plain --suite=TestFindInvalidUTF8
+     ./runtests --format=plain --suite=TestUTF8QuotedStr
 }
 unit TestLazUTF8;
 
@@ -26,6 +27,7 @@ type
     procedure TestUTF8Trim;
     procedure TestUTF8Pos;
     procedure TestFindInvalidUTF8;
+    procedure TestUTF8QuotedStr;
   end;
 
 implementation
@@ -103,6 +105,24 @@ begin
   t(#$f0#$8f#$bf#$bf,0,'invalid: $ffff encoded as 4 byte');
 end;
 
+procedure TTestLazUTF8.TestUTF8QuotedStr;
+
+  procedure t(const S, Quote, Expected: string);
+  var
+    Actual: String;
+  begin
+    Actual:=UTF8QuotedStr(S,Quote);
+    AssertEquals('S="'+S+'" Quote="'+Quote+'"',Expected,Actual);
+  end;
+
+begin
+  t('','=','==');
+  t('','AB','ABAB');
+  t('A','A','AAAA');
+  t('bAb','A','AbAAbA');
+  t('cABc','AB','ABcABABcAB');
+end;
+
 initialization
   AddToLazUtilsTestSuite(TTestLazUTF8);
 

test/lazutils/testlazutils.pas

@@ -15,7 +15,7 @@ unit TestLazUtils;
 interface
 
 uses
-  Classes, SysUtils, fpcunit, testglobals, LazLogger, LazFileUtils;
+  Classes, SysUtils, fpcunit, testglobals, LazLogger, LazUTF8, LazFileUtils;
 
 type