codetools: finding and removing duplicate strings in define trees

git-svn-id: trunk@19913 -

components/codetools/codetoolsstructs.pas

@@ -141,6 +141,19 @@ type
   public
     constructor Create(CaseInsensitive: boolean);
   end;
+
+  { TStringTree }
+
+  TStringTree = class
+  public
+    Tree: TAVLTree;
+    constructor Create;
+    destructor Destroy; override;
+    procedure Clear;
+    function FindNode(const s: string): TAVLTreeNode; inline;
+    procedure ReplaceString(var s: string);
+    function CalcMemSize: PtrUInt;
+  end;
   
 function CompareStringToStringItems(Data1, Data2: Pointer): integer;
 function CompareStringAndStringToStringTreeItem(Key, Data: Pointer): integer;
@@ -154,6 +167,7 @@ function CompareFilenameAndFilenameToStringTreeItem(Key, Data: Pointer): integer
 function CompareFilenameToStringItemsI(Data1, Data2: Pointer): integer;
 function CompareFilenameAndFilenameToStringTreeItemI(Key, Data: Pointer): integer;
 
+function CompareAnsiStringPtrs(Data1, Data2: Pointer): integer;
 
 implementation
 
@@ -204,6 +218,11 @@ begin
                                      PStringToStringTreeItem(Data)^.Name);
 end;
 
+function CompareAnsiStringPtrs(Data1, Data2: Pointer): integer;
+begin
+  Result:=CompareStr(AnsiString(Data1),AnsiString(Data2));
+end;
+
 { TCodeXYPositions }
 
 function TCodeXYPositions.GetItems(Index: integer): PCodeXYPosition;
@@ -539,5 +558,65 @@ begin
                     @CompareFilenameAndFilenameToStringTreeItem);
 end;
 
+{ TStringTree }
+
+constructor TStringTree.Create;
+begin
+  Tree:=TAVLTree.Create(@CompareAnsiStringPtrs);
+end;
+
+destructor TStringTree.Destroy;
+begin
+  Clear;
+  FreeAndNil(Tree);
+  inherited Destroy;
+end;
+
+procedure TStringTree.Clear;
+var
+  Node: TAVLTreeNode;
+begin
+  Node:=Tree.FindLowest;
+  while Node<>nil do begin
+    AnsiString(Node.Data):='';
+    Node:=Tree.FindSuccessor(Node);
+  end;
+  Tree.Clear;
+end;
+
+function TStringTree.FindNode(const s: string): TAVLTreeNode;
+begin
+  Result:=Tree.Find(Pointer(s));
+end;
+
+procedure TStringTree.ReplaceString(var s: string);
+var
+  Node: TAVLTreeNode;
+  h: String;
+begin
+  Node:=FindNode(s);
+  if Node=nil then begin
+    // increase refcount
+    h:=s;
+    Tree.Add(Pointer(h));
+    Pointer(h):=nil; // keep refcount
+  end else
+    s:=AnsiString(Node.Data);
+end;
+
+function TStringTree.CalcMemSize: PtrUInt;
+var
+  Node: TAVLTreeNode;
+begin
+  Result:=PtrUInt(InstanceSize)
+    +PtrUInt(Tree.InstanceSize)
+    +PtrUInt(TAVLTreeNode.InstanceSize)*PtrUInt(Tree.Count);
+  Node:=Tree.FindLowest;
+  while Node<>nil do begin
+    inc(Result,MemSizeString(AnsiString(Node.Data)));
+    Node:=Tree.FindSuccessor(Node);
+  end;
+end;
+
 end.
 

components/codetools/definetemplates.pas

@@ -55,7 +55,7 @@ interface
 
 uses
   Classes, SysUtils, CodeToolsStrConsts, ExprEval, DirectoryCacher,
-  Laz_XMLCfg, AVL_Tree,
+  Laz_XMLCfg, AVL_Tree, CodeToolsStructs,
   Process, KeywordFuncLists, FileProcs;
 
 const
@@ -354,6 +354,7 @@ type
     FDirectoryCachePool: TCTDirectoryCachePool;
     FFirstDefineTemplate: TDefineTemplate;
     FCache: TAVLTree; // tree of TDirectoryDefines
+    FDefineStrings: TStringTree;
     FChangeStep: integer;
     FErrorDescription: string;
     FErrorTemplate: TDefineTemplate;
@@ -368,6 +369,7 @@ type
     function Calculate(DirDef: TDirectoryDefines): boolean;
     procedure IncreaseChangeStep;
     procedure SetDirectoryCachePool(const AValue: TCTDirectoryCachePool);
+    procedure RemoveDoubles(Defines: TDirectoryDefines);
   protected
     function FindDirectoryInCache(const Path: string): TDirectoryDefines;
     function GetDirDefinesForDirectory(const Path: string;
@@ -1821,7 +1823,7 @@ begin
   Stats.Add('TDirectoryDefines',PtrUInt(InstanceSize)
     +MemSizeString(Path));
   if Values<>nil then
-    Stats.Add('TDirectoryDefines.Values',Values.CalcMemSize);
+    Stats.Add('TDirectoryDefines.Values',Values.CalcMemSize(false,nil));
 end;
 
 
@@ -1888,7 +1890,8 @@ begin
   inherited Create;
   FFirstDefineTemplate:=nil;
   FCache:=TAVLTree.Create(@CompareDirectoryDefines);
-  
+  FDefineStrings:=TStringTree.Create;
+
   FMacroFunctions:=TKeyWordFunctionList.Create;
   FMacroFunctions.AddExtended('Ext',nil,@MacroFuncExtractFileExt);
   FMacroFunctions.AddExtended('PATH',nil,@MacroFuncExtractFilePath);
@@ -1904,6 +1907,7 @@ begin
   FMacroVariables.Free;
   FMacroFunctions.Free;
   FCache.Free;
+  FreeAndNil(FDefineStrings);
   inherited Destroy;
 end;
 
@@ -1953,6 +1957,7 @@ begin
       //DebugLn('[TDefineTree.GetDirDefinesForDirectory] B ',ExpPath,' ');
       if Calculate(Result) then begin
         //DebugLn('[TDefineTree.GetDirDefinesForDirectory] C success');
+        RemoveDoubles(Result);
         FCache.Add(Result);
       end else begin
         //DebugLn('[TDefineTree.GetDirDefinesForDirectory] D failed');
@@ -1974,6 +1979,7 @@ begin
     FVirtualDirCache.Path:=VirtualDirectory;
     if Calculate(FVirtualDirCache) then begin
       //DebugLn('TDefineTree.GetDirDefinesForVirtualDirectory ');
+      RemoveDoubles(FVirtualDirCache);
     end else begin
       FVirtualDirCache.Free;
       FVirtualDirCache:=nil;
@@ -2029,6 +2035,7 @@ begin
     FVirtualDirCache:=nil;
   end;
   IncreaseChangeStep;
+  FDefineStrings.Clear;
 end;
 
 procedure TDefineTree.DoPrepareTree;
@@ -2621,6 +2628,12 @@ begin
   FDirectoryCachePool:=AValue;
 end;
 
+procedure TDefineTree.RemoveDoubles(Defines: TDirectoryDefines);
+begin
+  if Defines=nil then exit;
+  Defines.Values.RemoveDoubles(@FDefineStrings.ReplaceString);
+end;
+
 procedure TDefineTree.Add(ADefineTemplate: TDefineTemplate);
 // add as last
 var LastDefTempl: TDefineTemplate;
@@ -2797,6 +2810,8 @@ begin
     FFirstDefineTemplate.CalcMemSize(Stats);
   if FVirtualDirCache<>nil then
     FVirtualDirCache.CalcMemSize(Stats);
+  if FDefineStrings<>nil then
+    Stats.Add('TDefineTree.FDefineStrings',FDefineStrings.CalcMemSize);
   if FCache<>nil then begin
     Stats.Add('TDefineTree.FCache.Count',FCache.Count);
     Node:=FCache.FindLowest;

components/codetools/expreval.pas

@@ -40,7 +40,7 @@ const
 
 type
   TOnValuesChanged = procedure of object;
-  
+  TOnGetSameString = procedure(var s: string) of object;
   ArrayOfAnsiString = ^AnsiString;
   
 
@@ -85,9 +85,10 @@ type
     function AsString: string;
     constructor Create;
     destructor Destroy; override;
+    procedure RemoveDoubles(OnGetSameString: TOnGetSameString);
     procedure ConsistencyCheck;
     procedure WriteDebugReport;
-    function CalcMemSize: PtrUInt;
+    function CalcMemSize(WithNamesAndValues: boolean = true; Original: TExpressionEvaluator = nil): PtrUInt;
     property ChangeStamp: integer read FChangeStamp;
     procedure IncreaseChangeStamp; inline;
   end;
@@ -224,6 +225,16 @@ begin
   inherited Destroy;
 end;
 
+procedure TExpressionEvaluator.RemoveDoubles(OnGetSameString: TOnGetSameString);
+var
+  i: Integer;
+begin
+  for i:=0 to FCount-1 do begin
+    OnGetSameString(FNames[i]);
+    OnGetSameString(FValues[i]);
+  end;
+end;
+
 function TExpressionEvaluator.Eval(const Expression: string): string;
 //  1 = true
 //  0 = syntax error
@@ -757,16 +768,26 @@ begin
   ConsistencyCheck;
 end;
 
-function TExpressionEvaluator.CalcMemSize: PtrUInt;
+function TExpressionEvaluator.CalcMemSize(WithNamesAndValues: boolean;
+  Original: TExpressionEvaluator): PtrUInt;
 var
   i: Integer;
+  j: LongInt;
 begin
   Result:=PtrUInt(InstanceSize)
     +MemSizeString(Expr)
     +SizeOf(Pointer)*FCount*2;
-  for i:=0 to FCount-1 do begin
+  if WithNamesAndValues then begin
-    inc(Result,MemSizeString(FNames[i]));
+    for i:=0 to FCount-1 do begin
-    inc(Result,MemSizeString(FValues[i]));
+      if Original<>nil then begin
+        j:=Original.IndexOfName(FNames[i],false);
+        if j>=0 then begin
+          if Pointer(FNames[i])=Pointer(Original.FNames[j]) then continue;
+        end;
+      end;
+      inc(Result,MemSizeString(FNames[i]));
+      inc(Result,MemSizeString(FValues[i]));
+    end;
   end;
 end;
 

components/codetools/linkscanner.pas

@@ -1416,7 +1416,7 @@ begin
       FLinkCapacity*SizeOf(TSourceLink));
   if FInitValues<>nil then
     Stats.Add('TLinkScanner.FInitValues',
-      FInitValues.CalcMemSize);
+      FInitValues.CalcMemSize(false)); // FInitValues are copies of strings of TDefineTree
   if FSourceChangeSteps<>nil then
     Stats.Add('TLinkScanner.FSourceChangeSteps',
       FSourceChangeSteps.InstanceSize
@@ -1426,7 +1426,7 @@ begin
       FIncludeStack.InstanceSize+FIncludeStack.Capacity*SizeOf(TSourceLink));
   if Values<>nil then
     Stats.Add('TLinkScanner.Values',
-      Values.CalcMemSize);
+      Values.CalcMemSize(true,FInitValues));
   if FMissingIncludeFiles<>nil then
     Stats.Add('TLinkScanner.FMissingIncludeFiles',
       FMissingIncludeFiles.InstanceSize);