wiki test: search on idle

git-svn-id: trunk@35741 -
This commit is contained in:
mattias 2012-03-05 16:28:59 +00:00
parent 7472cd20a3
commit dbab7ecefd
3 changed files with 293 additions and 104 deletions

View File

@ -170,21 +170,35 @@ type
Help: TWikiHelp;
end;
TWikiHelpProgressStep = (
whpsNone,
whpsWikiScanDir,
whpsWikiLoadPages,
whpsWikiExtractPageTexts,
whpsWikiLoadComplete,
whpsWikiSearch,
whpsWikiSearchComplete
);
{ TWikiHelp }
TWikiHelp = class(TComponent)
private
FAbortingLoad: boolean;
FAborting: boolean;
FConverter: TWiki2HelpConverter;
FMaxResults: integer;
FOnScanned: TNotifyEvent;
FOnSearched: TNotifyEvent;
FQuery: TWikiHelpQuery;
FLoadComplete: boolean;
FLoading: boolean;
FScoring: TWHScoring;
FXMLDirectory: string;
FCritSec: TRTLCriticalSection;
FScanThread: TWikiHelpThread;
fProgressStep: TWikiHelpProgressStep;
fProgressCount: integer;
fProgressMax: integer;
fWikiLoadTimeMSec: integer;
fWikiSearchTimeMSec: integer;
function GetImagesDirectory: string;
procedure SetImagesDirectory(AValue: string);
procedure SetMaxResults(AValue: integer);
@ -193,23 +207,25 @@ type
procedure EnterCritSect;
procedure LeaveCritSect;
procedure Scanned;
procedure DoSearch;
function FoundNodeToHTMLSnippet(aPage: TW2HelpPage; aNode: TWHTextNode;
aQuery: TWikiHelpQuery): string;
public
constructor Create(AOwner: TComponent); override;
destructor Destroy; override;
function GetProgressCaption: string;
function Busy: boolean;
// load wiki files
procedure StartLoading; // returns immediately
property Loading: boolean read FLoading;
function LoadingContent: boolean;
procedure AbortLoading(Wait: boolean);
property AbortingLoad: boolean read FAbortingLoad;
property LoadComplete: boolean read FLoadComplete;
property Aborting: boolean read FAborting;
function LoadComplete: boolean;
// search
procedure Search(const Term: string; const Languages: string = '');
procedure Search(aQuery: TWikiHelpQuery);
procedure TestSearch;
property Query: TWikiHelpQuery read FQuery;
property Scoring: TWHScoring read FScoring;
property MaxResults: integer read FMaxResults write SetMaxResults;
@ -218,6 +234,7 @@ type
property ImagesDirectory: string read GetImagesDirectory write SetImagesDirectory; // directory where the wiki image files are
property Converter: TWiki2HelpConverter read FConverter;
property OnScanned: TNotifyEvent read FOnScanned write FOnScanned;
property OnSearched: TNotifyEvent read FOnSearched write FOnSearched;
end;
var
@ -533,6 +550,8 @@ begin
Src:=TWikiHelpQuery(Obj);
if not Phrases.Equals(Src.Phrases) then exit;
// LoPhrases is computed from Phrases
if Languages<>Src.Languages then exit;
Result:=true;
end;
{ TW2HelpPage }
@ -933,9 +952,15 @@ var
begin
StartIndex:=Index*PagesPerThread;
EndIndex:=Min(StartIndex+PagesPerThread-1,Count-1);
if Help.AbortingLoad then exit;
if Help.Aborting then exit;
for i:=StartIndex to EndIndex do
ExtractPageText(TW2HelpPage(Pages[i]));
Help.EnterCritSect;
try
inc(Help.fProgressCount,PagesPerThread);
finally
Help.LeaveCritSect;
end;
end;
procedure TWiki2HelpConverter.ParallelLoadPage(Index: PtrInt; Data: Pointer;
@ -948,7 +973,7 @@ begin
StartIndex:=Index*PagesPerThread;
EndIndex:=Min(StartIndex+PagesPerThread-1,Count-1);
for i:=StartIndex to EndIndex do begin
if Help.AbortingLoad then exit;
if Help.Aborting then exit;
Page:=TW2HelpPage(Pages[i]);
try
Page.ParseWikiDoc(false);
@ -958,6 +983,12 @@ begin
end;
end;
end;
Help.EnterCritSect;
try
inc(Help.fProgressCount,PagesPerThread);
finally
Help.LeaveCritSect;
end;
end;
procedure TWiki2HelpConverter.ParallelComputeScores(Index: PtrInt;
@ -969,11 +1000,17 @@ var
begin
StartIndex:=Index*PagesPerThread;
EndIndex:=Min(StartIndex+PagesPerThread-1,Count-1);
if Help.AbortingLoad then exit;
if Help.Aborting then exit;
for i:=StartIndex to EndIndex do begin
Page:=TW2HelpPage(Pages[i]);
Page.Score:=Page.GetScore(FCurQuery,FCurScoring);
end;
Help.EnterCritSect;
try
inc(Help.fProgressCount,PagesPerThread);
finally
Help.LeaveCritSect;
end;
end;
procedure TWiki2HelpConverter.ExtractPageText(Page: TW2HelpPage);
@ -1010,6 +1047,14 @@ end;
procedure TWiki2HelpConverter.ExtractAllTexts;
begin
Help.EnterCritSect;
try
Help.fProgressStep:=whpsWikiExtractPageTexts;
Help.fProgressCount:=0;
Help.fProgressMax:=Count;
finally
Help.LeaveCritSect;
end;
ProcThreadPool.DoParallel(@ParallelExtractPageText,0,(Count-1) div PagesPerThread);
end;
@ -1019,6 +1064,14 @@ var
i: Integer;
Page: TW2HelpPage;
begin
Help.EnterCritSect;
try
Help.fProgressStep:=whpsWikiSearch;
Help.fProgressCount:=0;
Help.fProgressMax:=Count;
finally
Help.LeaveCritSect;
end;
FCurQuery:=Query;
FCurScoring:=Scoring;
if FoundPages=nil then
@ -1034,6 +1087,14 @@ end;
procedure TWiki2HelpConverter.LoadPages;
begin
Help.EnterCritSect;
try
Help.fProgressStep:=whpsWikiLoadPages;
Help.fProgressCount:=0;
Help.fProgressMax:=Count;
finally
Help.LeaveCritSect;
end;
ProcThreadPool.DoParallel(@ParallelLoadPage,0,(Count-1) div PagesPerThread);
end;
@ -1095,21 +1156,22 @@ begin
Filename:=Help.XMLDirectory+Files[i];
Help.Converter.AddWikiPage(Filename,false);
end;
if Help.AbortingLoad then exit;
if Help.Aborting then exit;
// load xml files
Help.Converter.LoadPages;
if Help.AbortingLoad then exit;
if Help.Aborting then exit;
// extract texts
Help.Converter.ConvertInit;
if Help.AbortingLoad then exit;
if Help.Aborting then exit;
Help.Converter.ExtractAllTexts;
if Help.AbortingLoad then exit;
if Help.Aborting then exit;
fCompleted:=true;
EndTime:=Now;
Log('TWikiHelpThread.Execute SCAN complete XMLDirectory="'+Help.XMLDirectory+'" '+dbgs(round(Abs(EndTime-StartTime)*86400000))+'msec');
Help.fWikiLoadTimeMSec:=round(Abs(EndTime-StartTime)*86400000);
Log('TWikiHelpThread.Execute SCAN complete XMLDirectory="'+Help.XMLDirectory+'" '+dbgs(Help.fWikiLoadTimeMSec)+'msec');
finally
Files.Free;
Help.Converter.OnLog:=nil;
@ -1150,9 +1212,10 @@ begin
Help.EnterCritSect;
try
Help.FScanThread:=nil;
Help.FLoading:=false;
Help.FLoadComplete:=fCompleted;
Help.FAbortingLoad:=false;
if fCompleted then
Help.fProgressStep:=whpsWikiLoadComplete
else
Help.fProgressStep:=whpsNone;
finally
Help.LeaveCritSect;
end;
@ -1212,10 +1275,50 @@ procedure TWikiHelp.Scanned;
begin
if Assigned(OnScanned) then
OnScanned(Self);
DoSearch;
end;
{$IFDEF TestWikiSearch}
Search('documentation');
{$ENDIF}
procedure TWikiHelp.DoSearch;
var
StartTime: TDateTime;
EndTime: TDateTime;
FoundPages: TFPList;
i: Integer;
Page: TW2HelpPage;
Node: TWHTextNode;
s: String;
begin
if Query.Phrases.Count=0 then begin
EnterCritSect;
try
fProgressStep:=whpsWikiLoadComplete;
finally
LeaveCritSect;
end;
exit;
end;
StartTime:=Now;
//debugln(['TWikiHelp.DoSearch START Search=',Trim(Query.Phrases.Text)]);
FoundPages:=nil;
Converter.Search(Query,Scoring,FoundPages);
for i:=0 to Min(FoundPages.Count-1,MaxResults) do begin
Page:=TW2HelpPage(FoundPages[i]);
Node:=Page.GetNodeHighestScore(Query,Scoring);
s:=FoundNodeToHTMLSnippet(Page,Node,Query);
//debugln(['TWikiHelp.TestSearch Score=',Page.Score,' HTML="',s,'"']);
end;
FoundPages.Free;
EndTime:=Now;
fWikiSearchTimeMSec:=round(Abs(EndTime-StartTime)*86400000);
EnterCritSect;
try
fProgressStep:=whpsWikiSearchComplete;
finally
LeaveCritSect;
end;
//debugln(['TWikiHelp.DoSearch END Search="',Trim(Query.Phrases.Text),'" ',dbgs(fWikiSearchTimeMSec)+'msec']);
if Assigned(OnSearched) then
OnSearched(Self);
end;
function TWikiHelp.FoundNodeToHTMLSnippet(aPage: TW2HelpPage;
@ -1259,6 +1362,7 @@ begin
FScoring.Phrases[whfcLink,whfsWholeWord]:=2;
FScoring.Phrases[whfcLink,whfsPart]:=1;
FMaxResults:=10;
fProgressStep:=whpsNone;
end;
destructor TWikiHelp.Destroy;
@ -1280,9 +1384,11 @@ begin
raise Exception.Create('TWikiHelp.StartScan ImagesDirectory not found: '+ImagesDirectory);
EnterCritSect;
try
if Loading then exit;
FLoading:=true;
FLoadComplete:=false;
if fProgressStep>whpsNone then exit;
fProgressStep:=whpsWikiScanDir;
fWikiLoadTimeMSec:=0;
fProgressCount:=0;
fProgressMax:=0;
FScanThread:=TWikiHelpThread.Create(true);
FScanThread.FreeOnTerminate:=true;
FScanThread.Help:=Self;
@ -1296,26 +1402,60 @@ begin
end;
end;
function TWikiHelp.LoadingContent: boolean;
begin
Result:=(fProgressStep>whpsNone) and (fProgressStep<whpsWikiLoadComplete);
end;
procedure TWikiHelp.AbortLoading(Wait: boolean);
begin
EnterCritSect;
try
if not Loading then exit;
FAbortingLoad:=true;
if not LoadingContent then exit;
FAborting:=true;
finally
LeaveCritSect;
end;
if not Wait then exit;
while Loading do
while LoadingContent do
Sleep(10);
EnterCritSect;
try
FAbortingLoad:=false;
FAborting:=false;
finally
LeaveCritSect;
end;
end;
function TWikiHelp.LoadComplete: boolean;
begin
Result:=(fProgressStep>=whpsWikiLoadComplete);
end;
function TWikiHelp.GetProgressCaption: string;
begin
EnterCritSect;
try
case fProgressStep of
whpsNone: Result:='Wiki not yet loaded.';
whpsWikiScanDir: Result:='Scanning Wiki directory ...';
whpsWikiLoadPages: Result:='Loaded '+IntToStr(fProgressCount)+' of '+IntToStr(fProgressMax)+' Wiki pages.';
whpsWikiExtractPageTexts: Result:='Read '+IntToStr(fProgressCount)+' of '+IntToStr(fProgressMax)+' Wiki pages.';
whpsWikiLoadComplete: Result:='Loaded '+IntToStr(Converter.Count)+' Wiki pages in '+IntToStr(fWikiLoadTimeMSec)+'msec.';
whpsWikiSearch: Result:='Searched '+IntToStr(fProgressCount)+' of '+IntToStr(fProgressMax)+' Wiki pages.';
whpsWikiSearchComplete: Result:='Searched '+IntToStr(Converter.Count)+' Wiki pages in '+IntToStr(fWikiSearchTimeMSec)+'msec.';
else Result:='unknown step: '+IntToStr(ord(fProgressStep));
end;
finally
LeaveCritSect;
end;
end;
function TWikiHelp.Busy: boolean;
begin
Result:=not (fProgressStep in [whpsWikiLoadComplete,whpsWikiSearchComplete]);
end;
procedure TWikiHelp.Search(const Term: string; const Languages: string);
begin
Search(TWikiHelpQuery.Create(Term,Languages));
@ -1332,37 +1472,11 @@ begin
end;
FreeAndNil(FQuery);
FQuery:=aQuery;
if LoadingContent then exit;
finally
LeaveCritSect;
end;
TestSearch;
end;
procedure TWikiHelp.TestSearch;
var
StartTime: TDateTime;
EndTime: TDateTime;
FoundPages: TFPList;
i: Integer;
Page: TW2HelpPage;
Node: TWHTextNode;
s: String;
begin
StartTime:=Now;
debugln(['TWikiHelp.TestSearch START Search=',Trim(Query.Phrases.Text)]);
FoundPages:=nil;
Converter.Search(Query,Scoring,FoundPages);
for i:=0 to Min(FoundPages.Count-1,MaxResults) do begin
Page:=TW2HelpPage(FoundPages[i]);
debugln('===============================================');
debugln(['TWikiHelp.TestSearch ',Page.WikiDocumentName,' ',Page.Score]);
Node:=Page.GetNodeHighestScore(Query,Scoring);
s:=FoundNodeToHTMLSnippet(Page,Node,Query);
debugln(['TWikiHelp.TestSearch Score=',Page.Score,' HTML="',s,'"']);
end;
FoundPages.Free;
EndTime:=Now;
debugln(['TWikiHelp.TestSearch END Search="',Trim(Query.Phrases.Text),'" ',dbgs(round(Abs(EndTime-StartTime)*86400000))+'msec']);
DoSearch;
end;
end.

View File

@ -1,7 +1,7 @@
object WikiSearchDemoForm: TWikiSearchDemoForm
Left = 734
Left = 605
Height = 501
Top = 312
Top = 233
Width = 663
Caption = 'WikiSearchDemoForm'
ClientHeight = 501
@ -35,29 +35,30 @@ object WikiSearchDemoForm: TWikiSearchDemoForm
Anchors = [akTop, akLeft, akRight]
BorderSpacing.Around = 6
OnChange = SearchEditChange
ParentShowHint = False
ShowHint = True
TabOrder = 0
Text = 'SearchEdit'
end
object ResultsGroupBox: TGroupBox
object MainGroupBox: TGroupBox
AnchorSideTop.Control = LanguagesEdit
AnchorSideTop.Side = asrBottom
Left = 6
Height = 260
Left = 0
Height = 435
Top = 66
Width = 651
Width = 663
Align = alBottom
Anchors = [akTop, akLeft, akRight, akBottom]
BorderSpacing.Around = 6
Caption = 'ResultsGroupBox'
ClientHeight = 244
ClientWidth = 647
Caption = 'MainGroupBox'
ClientHeight = 419
ClientWidth = 659
TabOrder = 1
object ResultsIpHtmlPanel: TIpHtmlPanel
Left = 0
Height = 244
Height = 419
Top = 0
Width = 647
Align = alClient
Width = 337
Align = alLeft
FixedTypeface = 'Courier New'
DefaultTypeFace = 'default'
DefaultFontSize = 12
@ -68,23 +69,17 @@ object WikiSearchDemoForm: TWikiSearchDemoForm
PrintSettings.MarginBottom = 0.5
TabOrder = 0
end
end
object PageGroupBox: TGroupBox
Left = 6
Height = 152
Top = 343
Width = 651
Align = alBottom
BorderSpacing.Around = 6
Caption = 'PageGroupBox'
ClientHeight = 136
ClientWidth = 647
TabOrder = 2
object Splitter1: TSplitter
Left = 337
Height = 419
Top = 0
Width = 5
end
object PageIpHtmlPanel: TIpHtmlPanel
Left = 0
Height = 136
Left = 342
Height = 419
Top = 0
Width = 647
Width = 317
Align = alClient
FixedTypeface = 'Courier New'
DefaultTypeFace = 'default'
@ -94,18 +89,9 @@ object WikiSearchDemoForm: TWikiSearchDemoForm
PrintSettings.MarginTop = 0.5
PrintSettings.MarginRight = 0.5
PrintSettings.MarginBottom = 0.5
TabOrder = 0
TabOrder = 2
end
end
object Splitter1: TSplitter
Cursor = crVSplit
Left = 0
Height = 5
Top = 332
Width = 663
Align = alBottom
ResizeAnchor = akBottom
end
object LanguagesLabel: TLabel
AnchorSideLeft.Control = Owner
AnchorSideTop.Control = LanguagesEdit
@ -128,7 +114,30 @@ object WikiSearchDemoForm: TWikiSearchDemoForm
Top = 36
Width = 80
BorderSpacing.Around = 6
TabOrder = 4
OnChange = LanguagesEditChange
ParentShowHint = False
ShowHint = True
TabOrder = 2
Text = 'LanguagesEdit'
end
object ProgressLabel: TLabel
AnchorSideLeft.Side = asrBottom
AnchorSideTop.Control = LanguagesEdit
AnchorSideTop.Side = asrCenter
AnchorSideRight.Control = Owner
AnchorSideRight.Side = asrBottom
Left = 578
Height = 15
Top = 41
Width = 79
Anchors = [akTop, akRight]
BorderSpacing.Right = 6
Caption = 'ProgressLabel'
ParentColor = False
end
object Timer1: TTimer
OnTimer = Timer1Timer
left = 380
top = 40
end
end

View File

@ -5,8 +5,8 @@ unit WikiSearchMain;
interface
uses
Classes, SysUtils, FileUtil, LazLogger, IpHtml, Forms, Controls, Graphics,
Dialogs, StdCtrls, ExtCtrls, WikiHelpManager;
Classes, SysUtils, FileUtil, LazLogger, LazUTF8, IpHtml, Forms, Controls,
Graphics, Dialogs, StdCtrls, ExtCtrls, WikiHelpManager;
type
@ -15,19 +15,31 @@ type
TWikiSearchDemoForm = class(TForm)
LanguagesEdit: TEdit;
LanguagesLabel: TLabel;
PageGroupBox: TGroupBox;
PageIpHtmlPanel: TIpHtmlPanel;
ResultsGroupBox: TGroupBox;
ProgressLabel: TLabel;
MainGroupBox: TGroupBox;
ResultsIpHtmlPanel: TIpHtmlPanel;
SearchEdit: TEdit;
SearchLabel: TLabel;
Splitter1: TSplitter;
Timer1: TTimer;
procedure FormCreate(Sender: TObject);
procedure FormDestroy(Sender: TObject);
procedure LanguagesEditChange(Sender: TObject);
procedure OnIdle(Sender: TObject; var {%H-}Done: Boolean);
procedure SearchEditChange(Sender: TObject);
procedure Timer1Timer(Sender: TObject);
procedure WikiHelpScanned(Sender: TObject);
procedure WikiHelpSearched(Sender: TObject);
private
fLastSearchText: string;
fLastLanguages: string;
FIdleConnected: boolean;
procedure SearchParamsChanged;
procedure SetIdleConnected(AValue: boolean);
procedure UpdateProgress;
public
property IdleConnected: boolean read FIdleConnected write SetIdleConnected;
end;
var
@ -43,17 +55,20 @@ procedure TWikiSearchDemoForm.FormCreate(Sender: TObject);
begin
Caption:='Search Wiki (Proof of concept)';
SearchLabel.Caption:='Search:';
SearchEdit.Text:='';
LanguagesLabel.Caption:='Languages: empty for only original, "de" for german too, "-,de" for german only';
SearchEdit.Text:='Documentation';
SearchEdit.Hint:='Type one or more words separated by space, use " for phrases with spaces';
LanguagesLabel.Caption:='Languages:';
LanguagesEdit.Text:='';
ResultsGroupBox.Caption:='Result:';
PageGroupBox.Caption:='Page:';
LanguagesEdit.Hint:='Empty for only original/untranslated pages, "de" to include german pages, "-,de" for german pages only';
MainGroupBox.Caption:='Result:';
WikiHelp:=TWikiHelp.Create(nil);
WikiHelp.XMLDirectory:=SetDirSeparators('../wikixml');
WikiHelp.ImagesDirectory:=SetDirSeparators('../images');
WikiHelp.Converter.OutputDir:='';
WikiHelp.Converter.CSSFilename:='wiki.css';
WikiHelp.OnScanned:=@WikiHelpScanned;
WikiHelp.OnSearched:=@WikiHelpSearched;
WikiHelp.StartLoading;
end;
@ -63,14 +78,65 @@ begin
FreeAndNil(WikiHelp);
end;
procedure TWikiSearchDemoForm.SearchEditChange(Sender: TObject);
procedure TWikiSearchDemoForm.LanguagesEditChange(Sender: TObject);
begin
IdleConnected:=true;
end;
procedure TWikiSearchDemoForm.OnIdle(Sender: TObject; var Done: Boolean);
begin
SearchParamsChanged;
IdleConnected:=false;
end;
procedure TWikiSearchDemoForm.SearchEditChange(Sender: TObject);
begin
IdleConnected:=true;
end;
procedure TWikiSearchDemoForm.Timer1Timer(Sender: TObject);
begin
ProgressLabel.Caption:=WikiHelp.GetProgressCaption;
Timer1.Enabled:=WikiHelp.Busy;
end;
procedure TWikiSearchDemoForm.WikiHelpScanned(Sender: TObject);
begin
UpdateProgress;
end;
procedure TWikiSearchDemoForm.WikiHelpSearched(Sender: TObject);
begin
UpdateProgress;
end;
procedure TWikiSearchDemoForm.SearchParamsChanged;
var
NewSearchText: String;
NewLanguages: String;
begin
NewSearchText:=UTF8Trim(SearchEdit.Text);
NewLanguages:=UTF8Trim(LanguagesEdit.Text);
if (NewSearchText=fLastSearchText) and (NewLanguages=fLastLanguages) then
exit;
WikiHelp.Search(NewSearchText,NewLanguages);
Timer1.Enabled:=true;
end;
procedure TWikiSearchDemoForm.SetIdleConnected(AValue: boolean);
begin
if FIdleConnected=AValue then Exit;
FIdleConnected:=AValue;
if IdleConnected then
Application.AddOnIdleHandler(@OnIdle)
else
Application.RemoveOnIdleHandler(@OnIdle);
end;
procedure TWikiSearchDemoForm.UpdateProgress;
begin
ProgressLabel.Caption:=WikiHelp.GetProgressCaption;
Timer1.Enabled:=WikiHelp.Busy;
end;
end.