From b292940e7d4be15669bfad782d2a77617df43e82 Mon Sep 17 00:00:00 2001 From: vincents Date: Mon, 18 Aug 2008 12:58:38 +0000 Subject: [PATCH] test results: started processing test results diff based on program/scripts from the FPC team git-svn-id: trunk@16119 - --- .gitattributes | 3 + test/testresult-db/createdb.sql | 2 +- test/testresult-db/mailtestresults.sh | 28 +++ test/testresult-db/proctestsuitediff.lpi | 164 +++++++++++++ test/testresult-db/proctestsuitediff.pp | 280 +++++++++++++++++++++++ 5 files changed, 476 insertions(+), 1 deletion(-) create mode 100755 test/testresult-db/mailtestresults.sh create mode 100644 test/testresult-db/proctestsuitediff.lpi create mode 100644 test/testresult-db/proctestsuitediff.pp diff --git a/.gitattributes b/.gitattributes index 72a01be4a1..b9b1933894 100644 --- a/.gitattributes +++ b/.gitattributes @@ -3751,6 +3751,9 @@ test/testresult-db/createdb.sql svneol=native#text/plain test/testresult-db/dbtests.pp svneol=native#text/plain test/testresult-db/importtestresults.lpi svneol=native#text/plain test/testresult-db/importtestresults.pp svneol=native#text/plain +test/testresult-db/mailtestresults.sh svneol=native#text/plain +test/testresult-db/proctestsuitediff.lpi svneol=native#text/plain +test/testresult-db/proctestsuitediff.pp svneol=native#text/plain test/testresult-db/teststr.pp svneol=native#text/plain test/testresult-db/testsuite.lpi svneol=native#text/plain test/testresult-db/testsuite.pp svneol=native#text/plain diff --git a/test/testresult-db/createdb.sql b/test/testresult-db/createdb.sql index c5ec76ade7..1572d737cf 100644 --- a/test/testresult-db/createdb.sql +++ b/test/testresult-db/createdb.sql @@ -50,7 +50,7 @@ DROP TABLE IF EXISTS `TESTFPCVERSION`; CREATE TABLE IF NOT EXISTS `TESTFPCVERSION` ( `TFV_ID` int(11) NOT NULL auto_increment, `TFV_VERSION` varchar(10) default NULL, - `TFV_RELEASEDATE` timestamp NOT NULL default CURRENT_TIMESTAMP on update CURRENT_TIMESTAMP, + `TFV_RELEASEDATE` timestamp NOT NULL default CURRENT_TIMESTAMP, PRIMARY KEY (`TFV_ID`), UNIQUE KEY `TFV_INAME` (`TFV_VERSION`) ) ENGINE=MyISAM DEFAULT CHARSET=latin1 AUTO_INCREMENT=6 ; diff --git a/test/testresult-db/mailtestresults.sh b/test/testresult-db/mailtestresults.sh new file mode 100755 index 0000000000..de1e63c742 --- /dev/null +++ b/test/testresult-db/mailtestresults.sh @@ -0,0 +1,28 @@ +#!/bin/sh + +PROCTESTSUITEDIFF=/home/lazarus/testsuite/bin/proctestsuitediff +MAILDIR=/home/lazarus/testsuite/mail +CFGFILE=$MAILDIR/mailtestresults.cfg + +cd $MAILDIR + +. $CFGFILE + +datename=`date +%Y-%m-%d` + +cat > tests_mail << EOF +Subject: Daily test suite diffs ($datename) +From: "Lazarus Testsuite Diff Cron" +To: "Lazarus Developer List" + +EOF +mysql -vvv -u ${USERNAME} --password=${PASSWORD} laz_testsuite -e ' +SELECT (TU_FAILURECOUNT+TU_ERRORCOUNT) as FAILS,DATE(TU_DATE) as DATE,TESTFPCVERSION.TFV_VERSION as FPCVERSION, + TESTCPU.TC_NAME as CPU,TESTOS.TO_NAME as OS, TESTWIDGETSET.TW_NAME as WIDGETSET, TESTOS.TO_NAME as OS, + TU_SUBMITTER as TESTER,TU_MACHINE as MACHINE,TU_COMMENT as COMMENT, TIME(TU_DATE) as TIME, TU_ID, GROUP_CONCAT(TR_TEST_FK) +FROM TESTRUN LEFT JOIN (TESTCPU) ON (TU_CPU_FK=TC_ID) LEFT JOIN (TESTOS) ON (TU_OS_FK=TO_ID) LEFT JOIN (TESTFPCVERSION) ON (TU_FPC_VERSION_FK=TFV_ID) + LEFT JOIN TESTRESULTS ON (TR_TESTRUN_FK=TU_ID) +WHERE (DATE_SUB(CURDATE(), INTERVAL 2 DAY)<=DATE(TU_DATE)) AND TR_OK<>"+" AND TR_SKIP<>"+" +GROUP BY TU_ID +ORDER BY FPCVERSION, OS, CPU, TESTER, MACHINE, COMMENT, DATE;' | tee mysql-output | $PROCTESTSUITEDIFF >> tests_mail +#/usr/sbin/sendmail -f ${MAILFROM} ${MAILTO} < tests_mail >/dev/null 2>&1 diff --git a/test/testresult-db/proctestsuitediff.lpi b/test/testresult-db/proctestsuitediff.lpi new file mode 100644 index 0000000000..c1dc596a26 --- /dev/null +++ b/test/testresult-db/proctestsuitediff.lpi @@ -0,0 +1,164 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/test/testresult-db/proctestsuitediff.pp b/test/testresult-db/proctestsuitediff.pp new file mode 100644 index 0000000000..7e1914ddb3 --- /dev/null +++ b/test/testresult-db/proctestsuitediff.pp @@ -0,0 +1,280 @@ +program proctestsuitediff; +{$mode objfpc}{$h+} + +uses + sysutils, classes, strutils; + +const + runhour = 8; { cut-off hour that distinguishes yesterday and today } + urlprefix = 'http://fpcfos64.freepascal.org/laztestsuite/chi-bin/testsuite.cgi?'; + +function getdate(line: string): string; +begin + result := copy(line, posex('|', line, Pos('|', line)+1)+2, 10); +end; + +function getfail(line: string): string; +var + i, j: integer; +begin + i := 2; + while (i < length(line)) and (line[i] = ' ') do + inc(i); + j := i+1; + while (j < length(line)) and (line[j] in ['0'..'9']) do + inc(j); + result := copy(line, i, j-i); +end; + +type + toutputline = class(tobject) + public + data, url: string; + end; + +var + header: array[0..2] of string; + footer: string; + lenfailstr: integer; + urllist: tstrings; + +procedure printtable(list: tstringlist; heading: string); +var + outputline: toutputline; + str, urlref: string; + i: integer; +begin + if list.count = 0 then + exit; + writeln(heading); + for i := 0 to 2 do + writeln(header[i]); + for i := 0 to list.count - 1 do + begin + str := list.strings[i]; + outputline := toutputline(list.objects[i]); + urlref := inttostr(urllist.count+1); + writeln('| ' + stringofchar(' ', 3 - length(urlref)) + urlref + ' | ' + str + stringofchar(' ', lenfailstr - length(str)) + ' ' + outputline.data); + urllist.add(outputline.url); + end; + writeln(footer); + writeln; +end; + +procedure printurllist; +var + i: integer; +begin + for i := 0 to urllist.count-1 do + writeln('[' + inttostr(i+1) + ']: ' + urllist.strings[i]); + writeln; +end; + +procedure addlist(list: tstrings; const failstr, data, url: string); +var + outputline: toutputline; +begin + outputline := toutputline.create; + outputline.data := data; + outputline.url := url; + list.addobject(failstr, outputline); + if length(failstr) > lenfailstr then + lenfailstr := length(failstr); +end; + +type + ttestrun = record + line, date, fail, data, runid, dbfail, failset: string; + hour: integer; + end; + +function construct_results_url(const runid: string): string; +begin + result := urlprefix+'action=1&failedonly=1&run1id='+runid; +end; + +function construct_compare_url(const run1id, run2id: string): string; +begin + result := urlprefix+'action=1&run1id='+run1id+'&run2id='+run2id+'&noskipped=1'; +end; + +function checkchange(var prev, curr: ttestrun; const prevdate, currdate: string; + changelist, nochangelist: tstrings): boolean; +var + failstr: string; +begin + result := (prev.date = prevdate) and (prev.data = curr.data) and (curr.date = currdate); + if result then + begin + if (length(curr.line) <> 0) and (length(prev.line) <> 0) then + begin + if prev.failset = curr.failset then + begin + failstr := curr.fail; + addlist(nochangelist, failstr, curr.data, construct_results_url(curr.runid)); + end else begin + failstr := prev.fail + ' -> ' + curr.fail; + addlist(changelist, failstr, curr.data, construct_compare_url(prev.runid, curr.runid)); + end; + end; + { both these lines have been processed } + curr.line := ''; + prev.line := ''; + end; +end; + +function findseparator(aoffset, aindex: integer): integer; +var + I: integer; +begin + for I := 1 to aindex do + begin + inc(aoffset); + while (aoffset '|') do + inc(aoffset); + end; + result := aoffset; +end; + +const + { cut fails and date (first two fields, '| FAILS | DATE ', 21 characters) } + datastart = 22; + +var + twodaysago, yesterday, today: string; + curr, prev, old: ttestrun; + list, prevnochangelist, prevchangelist, prevdisappearlist: tstringlist; + prevnewlist, disappearlist, nochangelist, changelist, newlist: tstringlist; + blinkerchangelist, blinkernochangelist: tstringlist; + todaydate: TDateTime; + dataend, datalen, houroffset: integer; + runidoffset, runidend, runidlen: integer; + dbfailsep, failoffset, failend: integer; +begin + blinkernochangelist := tstringlist.create; + blinkerchangelist := tstringlist.create; + prevdisappearlist := tstringlist.create; + prevnochangelist := tstringlist.create; + prevchangelist := tstringlist.create; + disappearlist := tstringlist.create; + nochangelist := tstringlist.create; + prevnewlist := tstringlist.create; + changelist := tstringlist.create; + newlist := tstringlist.create; + urllist := tstringlist.create; + footer := ''; + old.data := ''; + readln; + repeat + if eof then + halt(1); + readln(header[0]); + until (length(header[0]) > 0) and (header[0][1] = '+'); + readln(header[1]); + readln(header[2]); + if ParamCount >= 3 then + todaydate := EncodeDate(StrToInt(ParamStr(1)), StrToInt(ParamStr(2)), + StrToInt(ParamStr(3))) + else + todaydate := Now; + twodaysago := FormatDateTime('YYYY-mm-dd', todaydate-2); + yesterday := FormatDateTime('YYYY-mm-dd', todaydate-1); + today := FormatDateTime('YYYY-mm-dd', todaydate); + lenfailstr := 5; { Length('FAILS') = column header } + dataend := findseparator(datastart, 6); + datalen := dataend - datastart + 1; + { cut time (last 2 fields, ' HH:MM:SS | XXXX |') } + houroffset := dataend + 2; + runidoffset := houroffset + 11; + runidend := findseparator(runidoffset, 1); + runidlen := runidend - 1 - runidoffset; + failoffset := runidend + 2; + failend := findseparator(failoffset, 1); + fillchar(curr,sizeof(curr),0); + repeat + if eof then + break; + if (length(curr.line) = 0) or (curr.line[1] <> '+') then + begin + readln(curr.line); + curr.fail := getfail(curr.line); + curr.date := getdate(curr.line); + curr.data := copy(curr.line, datastart, datalen); + curr.hour := strtointdef(copy(curr.line, houroffset, 2), 0); + curr.runid := trim(copy(curr.line, runidoffset, runidlen)); + dbfailsep := posex('|', curr.line, failoffset); + curr.dbfail := copy(curr.line, failoffset, dbfailsep-failoffset); + curr.failset := trim(copy(curr.line, dbfailsep+1, failend-2-dbfailsep)); + if curr.dbfail <> curr.fail then + curr.fail := curr.fail + ' (' + curr.dbfail + ')'; + end else + if length(footer) = 0 then + footer := curr.line; + { 'same' testrun yesterday and today, changelist or nochangelist modified } + if checkchange(prev, curr, yesterday, today, changelist, nochangelist) + and (old.data = prev.data) then + old.line := ''; + { 'same' testrun two days ago and today, a "blinker" } + if checkchange(prev, curr, twodaysago, today, blinkerchangelist, blinkernochangelist) + and (old.data = prev.data) then + old.line := ''; + { 'same' testrun two days ago and yesterday, prevchangelist or prevnochangelist modified } + { only detect equal testruns yesterday if submitted late for diff mail yesterday } + if prev.hour >= runhour then + checkchange(old, prev, twodaysago, yesterday, prevchangelist, prevnochangelist); + { still some unprocessed line? } + if length(old.line) > 0 then + begin + list := nil; + if old.date = twodaysago then + begin + if old.hour >= runhour then + list := prevdisappearlist + { else we already had it disappear yesterday } + end else if old.date = yesterday then + if old.hour < runhour then + list := disappearlist + else + list := prevnewlist + else if old.date = today then + list := newlist; + if list <> nil then + addlist(list, old.fail, old.data, construct_results_url(old.runid)); + end; + old := prev; + prev := curr; + until (length(old.line) > 0) and (old.line[1] = '+'); + + header[0] := '+-----' + copy(header[0], 1, 1) + stringofchar('-', lenfailstr+2) + + copy(header[0], datastart, datalen); + header[1] := '| URL ' + copy(header[1], 1, 7) + stringofchar(' ', lenfailstr-4) + + copy(header[1], datastart, datalen); + header[2] := '+-----' + copy(header[2], 1, 1) + stringofchar('-', lenfailstr+2) + + copy(header[2], datastart, datalen); + footer := '+-----' + copy(footer, 1, 1) + stringofchar('-', lenfailstr+2) + + copy(footer, datastart, datalen); + + printtable(disappearlist, 'DISAPPEARED:'); + printtable(prevdisappearlist, 'DISAPPEARED YESTERDAY:'); + printtable(changelist, 'CHANGED:'); + printtable(prevchangelist, 'CHANGED YESTERDAY:'); + printtable(blinkerchangelist, 'CHANGED BLINKER:'); + printtable(newlist, 'NEW:'); + printtable(prevnewlist, 'NEW YESTERDAY:'); + printtable(nochangelist, 'UNCHANGED:'); + printtable(prevnochangelist, 'UNCHANGED YESTERDAY:'); + printtable(blinkernochangelist, 'UNCHANGED BLINKER:'); + + printurllist; + + newlist.free; + changelist.free; + prevnewlist.free; + nochangelist.free; + disappearlist.free; + prevchangelist.free; + prevnochangelist.free; + prevdisappearlist.free; + blinkerchangelist.free; + blinkernochangelist.free; +end.