codetools: c parser: fixed parsing til end of directive

git-svn-id: trunk@29591 -
This commit is contained in:
mattias 2011-02-17 16:57:54 +00:00
parent 2fe6028b41
commit dfbe1e665f
4 changed files with 134 additions and 147 deletions

View File

@ -104,7 +104,7 @@ interface
{$I codetools.inc}
{$DEFINE VerboseCCodeParser}
{off $DEFINE VerboseCDirectives}
{$DEFINE VerboseCDirectives}
uses
{$IFDEF MEM_CHECK}
@ -128,17 +128,17 @@ const
ccnEnumBlock = 4+ccnBase;// e.g. enum {};
ccnEnumID = 5+ccnBase;// e.g. name = value;
ccnConstant = 6+ccnBase;// e.g. 1
ccnTypedef = 7+ccnBase;// e.g. typedef int TInt;
ccnStruct = 8+ccnBase;// e.g. struct{}
ccnStructAlias = 9+ccnBase;// e.g. struct alias name;
ccnTypedef = 7+ccnBase;// e.g. typedef int TInt; last child is ccnName
ccnStruct = 8+ccnBase;// e.g. struct{}, child is ccnTypeName or ccnTypeName+ccnSubDefs or ccnSubDefs
ccnUnion = 10+ccnBase;// e.g. union{}
ccnUnionAlias = 11+ccnBase;// e.g. union alias name;
ccnVariable = 12+ccnBase;// e.g. int i
ccnDefinition = 12+ccnBase;// e.g. variable: int i, type: struct typename {} or both: union typename {} varname
ccnFunction = 13+ccnBase;// e.g. int i()
ccnName = 14+ccnBase;// e.g. i
ccnFuncParamList = 15+ccnBase;// e.g. ()
ccnFuncParameter = 16+ccnBase;// e.g. ()
ccnStatementBlock = 17+ccnBase;// e.g. {}
ccnTypeName = 15+ccnBase;// e.g. i
ccnSubDefs = 16+ccnBase;// e.g. the {} of a struct
ccnFuncParamList = 17+ccnBase;// e.g. ()
ccnFuncParameter = 18+ccnBase;// e.g. ()
ccnStatementBlock = 19+ccnBase;// e.g. {}
// values for Node.SubDesc
ccnsNone = 0;
@ -188,6 +188,7 @@ type
function CurlyBracketCloseToken: boolean;
function TypedefToken: boolean;
function StructToken: boolean;
function UnionToken: boolean;
procedure InitKeyWordList;
procedure InitParser;
@ -198,8 +199,7 @@ type
procedure ReadVariable(AsParameter: boolean);
procedure ReadParameterList;
procedure ReadEnum;
procedure ReadStruct;
procedure ReadUnion;
procedure ReadUnionStruct(IsStruct: boolean);
procedure ReadConstant;
procedure Read__attribute__;
@ -279,9 +279,9 @@ type
WithDirectives: boolean = false): string;// extract code without comments
function GetFirstNameNode(Node: TCodeTreeNode): TCodeTreeNode;
function ExtractVariableName(VarNode: TCodeTreeNode): string;
function ExtractVariableType(VarNode: TCodeTreeNode;
WithDirectives: boolean = false): string;
function ExtractDefinitionName(VarNode: TCodeTreeNode): string;
function ExtractDefinitionType(VarNode: TCodeTreeNode;
WithDirectives: boolean = false): string;
function ExtractFunctionName(FuncNode: TCodeTreeNode): string;
function GetFunctionParamListNode(Node: TCodeTreeNode): TCodeTreeNode;
function ExtractFunctionParamList(FuncNode: TCodeTreeNode): string;
@ -405,11 +405,11 @@ begin
ccnConstant : Result:='constant';
ccnTypedef : Result:='typedef';
ccnStruct : Result:='struct';
ccnStructAlias : Result:='struct-alias';
ccnUnion : Result:='union';
ccnVariable : Result:='variable';
ccnDefinition : Result:='definition(var/type/const)';
ccnFunction : Result:='function';
ccnName : Result:='name';
ccnTypeName : Result:='type-name';
ccnFuncParamList : Result:='function-param-list';
ccnFuncParameter : Result:='function-parameter';
ccnStatementBlock: Result:='statement-block';
@ -787,7 +787,10 @@ function TCCodeParserTool.DirectiveToken: boolean;
begin
BracketLevel:=0;
repeat
ReadRawNextCAtom(Src,SrcPos,AtomStart);
ReadRawNextAtom;
{$IFDEF VerboseCCodeParser}
debugln(['ReadExpression Atom ',GetAtom]);
{$ENDIF}
if AtomStart>SrcLen then
RaiseException('missing expression');
if Src[AtomStart] in [#10,#13] then begin
@ -811,17 +814,17 @@ function TCCodeParserTool.DirectiveToken: boolean;
if AtomIs('defined') then begin
// read defined(macro)
// or read defined macro
ReadRawNextCAtom(Src,SrcPos,AtomStart);
ReadRawNextAtom;
if AtomIsChar('(') then begin
NeedBracket:=true;
ReadRawNextCAtom(Src,SrcPos,AtomStart);
ReadRawNextAtom;
end else begin
NeedBracket:=false;
end;
if not AtomIsIdentifier then
RaiseExpectedButAtomFound('macro');
if NeedBracket then begin
ReadRawNextCAtom(Src,SrcPos,AtomStart);
ReadRawNextAtom;
if not AtomIsChar(')') then
RaiseExpectedButAtomFound(')');
end;
@ -937,8 +940,8 @@ begin
end;
end;
// read til end of line
ReadTilCLineEnd(Src,SrcPos);
AtomStart:=SrcPos;
ReadTilCLineEnd(Src,AtomStart);
SrcPos:=AtomStart;
//DebugLn(['TCCodeParserTool.DirectiveToken ',copy(Src,CurNode.StartPos,AtomStart-CurNode.Startpos)]);
EndChildNode;
end;
@ -1029,7 +1032,7 @@ begin
EndChildNode;
end;
procedure TCCodeParserTool.ReadStruct;
procedure TCCodeParserTool.ReadUnionStruct(IsStruct: boolean);
(* Examples:
union sign /* A definition and a declaration */
@ -1061,15 +1064,18 @@ procedure TCCodeParserTool.ReadStruct;
// typeof(*(ptr)) __v;
// } *__p = (void *) (ptr);
//
var
InTypeDef: Boolean;
begin
CreateChildNode(ccnStruct);
InTypeDef:=CurNode.Desc=ccnTypedef;
if IsStruct then
CreateChildNode(ccnStruct)
else
CreateChildNode(ccnUnion);
ReadNextAtom;
if CurNode.Parent.Desc<>ccnTypedef then begin
// read variable name
if not AtomIsIdentifier then
RaiseExpectedButAtomFound('identifier');
CreateChildNode(ccnName);
if AtomIsIdentifier then begin
CreateChildNode(ccnTypeName);
EndChildNode;
ReadNextAtom;
end;
@ -1081,6 +1087,7 @@ begin
end;
if AtomIsChar('{') then begin
// read block {}
CreateChildNode(ccnSubDefs);
repeat
ReadNextAtom;
// read variables
@ -1098,6 +1105,7 @@ begin
else
RaiseExpectedButAtomFound('identifier');
until false;
EndChildNode;
// read after attributes
ReadNextAtom;
if AtomIs('__attribute__') then begin
@ -1105,12 +1113,12 @@ begin
end else begin
UndoReadNextAtom;
end;
end else if AtomIsIdentifier then begin
end else if InTypeDef and AtomIsIdentifier then begin
// using another struct
CreateChildNode(ccnStructAlias);
EndChildNode;
UndoReadNextAtom;
end else if AtomIsChar(';') then begin
// struct without content
UndoReadNextAtom;
end else
RaiseExpectedButAtomFound('{');
@ -1118,74 +1126,6 @@ begin
EndChildNode;
end;
procedure TCCodeParserTool.ReadUnion;
(* Example
union {
uint16_t uuid16;
uint32_t uuid32;
uint128_t uuid128;
} value;
union _GFloatIEEE754
{
gfloat v_float;
struct {
guint mantissa : 23;
guint biased_exponent : 8;
guint sign : 1;
} mpn;
};
typedef union _GDoubleIEEE754 GDoubleIEEE754;
*)
begin
CreateChildNode(ccnUnion);
ReadNextAtom;
debugln(['TCCodeParserTool.ReadUnion AAA1 ',GetAtom]);
if AtomIsIdentifier then begin
// read type name
CreateChildNode(ccnName);
EndChildNode;
ReadNextAtom;
debugln(['TCCodeParserTool.ReadUnion AAA2 ',GetAtom]);
end;
if AtomIsChar('{') then begin
// read block {}
repeat
ReadNextAtom;
// read variables
if AtomIsIdentifier then begin
ReadVariable(false);
ReadNextAtom;
if AtomIsChar('}') then
break
else if AtomIsChar(';') then begin
// next identifier
end else
RaiseExpectedButAtomFound('}');
end else if AtomIsChar('}') then
break
else
RaiseExpectedButAtomFound('identifier');
until false;
end else if AtomIsIdentifier then begin
// using another union
// for example: typedef union _GDoubleIEEE754 GDoubleIEEE754;
CreateChildNode(ccnUnionAlias);
EndChildNode;
end else if AtomIsChar(';') then begin
// union without content
end else
RaiseExpectedButAtomFound('{');
debugln(['TCCodeParserTool.ReadUnion AAA3 ',GetAtom]);
// close node
EndChildNode;
end;
function TCCodeParserTool.TypedefToken: boolean;
{ examples:
typedef type name;
@ -1197,8 +1137,8 @@ begin
ReadNextAtom;
if AtomIs('typedef') then
RaiseExpectedButAtomFound('declaration')
else if AtomIs('struct') then begin
ReadStruct;
else if AtomIs('struct') or AtomIs('union') then begin
ReadUnionStruct(AtomIs('struct'));
ReadNextAtom;
if not AtomIsIdentifier then
RaiseExpectedButAtomFound('identifier');
@ -1225,7 +1165,13 @@ end;
function TCCodeParserTool.StructToken: boolean;
begin
Result:=true;
ReadStruct;
ReadUnionStruct(true);
end;
function TCCodeParserTool.UnionToken: boolean;
begin
Result:=true;
ReadUnionStruct(false);
end;
procedure TCCodeParserTool.InitKeyWordList;
@ -1239,6 +1185,7 @@ begin
Add('enum',{$ifdef FPC}@{$endif}EnumToken);
Add('typedef',{$ifdef FPC}@{$endif}TypedefToken);
Add('struct',{$ifdef FPC}@{$endif}StructToken);
Add('union',{$ifdef FPC}@{$endif}UnionToken);
DefaultKeyWordFunction:={$ifdef FPC}@{$endif}OtherToken;
end;
end;
@ -1379,17 +1326,15 @@ begin
exit;
end;
end else
CreateChildNode(ccnVariable);
CreateChildNode(ccnDefinition);
MainNode:=CurNode;
IsFunction:=false;
if AtomIs('const') then ReadNextAtom;
if AtomIs('struct') then begin
// for example: struct structname varname
ReadNextAtom;
end
else if AtomIs('union') then begin
ReadUnion;
ReadUnionStruct(true);
end else if AtomIs('union') then begin
ReadUnionStruct(false);
end else begin
if IsCCodeFunctionModifier.DoItCaseSensitive(Src,AtomStart,SrcPos-AtomStart)
then begin
@ -1834,6 +1779,9 @@ begin
end else begin
SrcPos:=AtomStart;
end;
{$IFDEF VerboseCCodeParser}
DebugLn(['TCCodeParserTool.UndoReadNextAtom END ',AtomStart,'-',SrcPos,' "',copy(Src,AtomStart,SrcPos-AtomStart),'"']);
{$ENDIF}
end;
function TCCodeParserTool.ReadTilBracketClose(
@ -2024,7 +1972,7 @@ begin
while (Result<>nil) and (Result.Desc<>ccnName) do Result:=Result.NextBrother;
end;
function TCCodeParserTool.ExtractVariableName(VarNode: TCodeTreeNode): string;
function TCCodeParserTool.ExtractDefinitionName(VarNode: TCodeTreeNode): string;
var
NameNode: TCodeTreeNode;
begin
@ -2035,7 +1983,7 @@ begin
Result:=copy(Src,NameNode.StartPos,NameNode.EndPos-NameNode.StartPos);
end;
function TCCodeParserTool.ExtractVariableType(VarNode: TCodeTreeNode;
function TCCodeParserTool.ExtractDefinitionType(VarNode: TCodeTreeNode;
WithDirectives: boolean): string;
var
NameNode: TCodeTreeNode;

View File

@ -33,7 +33,7 @@
<PackageName Value="CodeTools"/>
</Item2>
</RequiredPackages>
<Units Count="3">
<Units Count="4">
<Unit0>
<Filename Value="h2pastest.lpr"/>
<IsPartOfProject Value="True"/>
@ -47,6 +47,10 @@
<Filename Value="convert_glib.sh"/>
<IsPartOfProject Value="True"/>
</Unit2>
<Unit3>
<Filename Value="convert_glib3.sh"/>
<IsPartOfProject Value="True"/>
</Unit3>
</Units>
</ProjectOptions>
<CompilerOptions>

View File

@ -546,7 +546,7 @@ var
begin
if (CNode.FirstChild<>nil) and (CNode.FirstChild.Desc=ccnUnion)
then begin
CurName:=CTool.ExtractVariableName(CNode);
CurName:=CTool.ExtractDefinitionName(CNode);
if (ParentNode<>nil) and (ParentNode.PascalDesc=ctnRecordType)
then begin
// create a pascal 'record case'
@ -578,8 +578,8 @@ begin
DebugLn(['TH2PasTool.ConvertVariable SKIPPING union variable at ',CTool.CleanPosToStr(CNode.StartPos)]);
end;
end else begin
CurName:=CTool.ExtractVariableName(CNode);
CurType:=CTool.ExtractVariableType(CNode);
CurName:=CTool.ExtractDefinitionName(CNode);
CurType:=CTool.ExtractDefinitionType(CNode);
SimpleType:=GetSimplePascalTypeOfCVar(CNode);
if SimpleType='' then begin
// this variable has a complex type
@ -751,6 +751,8 @@ var
CurCName: String;
CurValue: String;
SubChildNode: TCodeTreeNode;
TypeNode: TCodeTreeNode;
SubTypeName: String;
begin
if CNode.FirstChild=nil then begin
exit;
@ -760,10 +762,10 @@ begin
ChildNode:=CNode.FirstChild;
case ChildNode.Desc of
ccnVariable: // typedef variable
ccnDefinition: // typedef name
begin
CurName:=CTool.ExtractVariableName(ChildNode);
CurType:=CTool.ExtractVariableType(ChildNode);
CurName:=CTool.ExtractDefinitionName(ChildNode);
CurType:=CTool.ExtractDefinitionType(ChildNode);
SimpleType:=GetSimplePascalTypeOfCVar(ChildNode);
if SimpleType='' then begin
// this variable has a complex type
@ -783,22 +785,49 @@ begin
ccnStruct: // typedef struct
begin
ChildNode:=CNode.FirstChild.FirstChild;
if (ChildNode<>nil)
and (ChildNode.Desc=ccnStructAlias) then begin
// this is a struct alias
CurType:=GetIdentifier(@CTool.Src[ChildNode.StartPos]);
CurCName:=CurName;
TypeH2PNode:=CreateH2PNode(CurName,CurCName,CNode,
ctnTypeDefinition,CurType);
end else begin
// this is a new struct
CurCName:=CurName;
TypeH2PNode:=CreateH2PNode(CurName,CurCName,CNode,ctnRecordType,'');
(* typedef struct a b; => alias b = a
typedef struct a {} b; => a = record + alias b = a
typedef struct {} b; => b = record
*)
if (ChildNode.FirstChild.Desc=ccnTypeName)
and (ChildNode.LastChild.Desc=ccnSubDefs) then begin
// for example: typedef struct a {} b;
// => create a new record type a and an alias b = a
TypeNode:=ChildNode.FirstChild;
SubChildNode:=ChildNode.LastChild;
// create a new record
CurCName:=GetIdentifier(@CTool.Src[TypeNode.StartPos]);
SubTypeName:=CurCName;
TypeH2PNode:=CreateH2PNode(SubTypeName,CurCName,TypeNode,ctnRecordType,'');
DebugLn(['TH2PasTool.ConvertTypedef added record: ',TypeH2PNode.DescAsString(CTool)]);
// build recursively
if ChildNode<>nil then
BuildH2PTree(TypeH2PNode,ChildNode);
BuildH2PTree(TypeH2PNode,SubChildNode.FirstChild);
// create an alias b=a
CurCName:=CurName;
TypeH2PNode:=CreateH2PNode(CurName,CurCName,CNode,
ctnTypeDefinition,SubTypeName);
DebugLn(['TH2PasTool.ConvertTypedef added type alias: ',TypeH2PNode.DescAsString(CTool)]);
end else if ChildNode.FirstChild.Desc=ccnSubDefs then begin
// for example: typedef struct {} b; => b = record
// => create a new record type b
SubChildNode:=ChildNode.LastChild;
CurCName:=CurName;
TypeH2PNode:=CreateH2PNode(CurName,CurCName,ChildNode,ctnRecordType,'');
DebugLn(['TH2PasTool.ConvertTypedef added record: ',TypeH2PNode.DescAsString(CTool)]);
// build recursively
BuildH2PTree(TypeH2PNode,SubChildNode.FirstChild);
end else if (ChildNode.FirstChild.Desc=ccnTypeName)
and (ChildNode.FirstChild.NextBrother=nil) then begin
// for example: typedef struct a b;
// => create a type alias b = a
TypeNode:=ChildNode.FirstChild;
SubTypeName:=GetIdentifier(@CTool.Src[TypeNode.StartPos]);
CurCName:=CurName;
TypeH2PNode:=CreateH2PNode(CurName,CurCName,CNode,
ctnTypeDefinition,SubTypeName);
DebugLn(['TH2PasTool.ConvertTypedef added type alias: ',TypeH2PNode.DescAsString(CTool)]);
end else begin
raise Exception.Create('TH2PasTool.ConvertTypedef inconsistency: unknown format of typedef struct');
end;
end;
@ -2008,7 +2037,7 @@ begin
ccnTypedef:
ConvertTypedef(CNode,ParentNode);
ccnVariable:
ccnDefinition:
ConvertVariable(CNode,ParentNode);
ccnFunction:
@ -2222,7 +2251,7 @@ end;
function TH2PasTool.GetSimplePascalTypeOfCVar(CVarNode: TCodeTreeNode): string;
begin
Result:=CTool.ExtractVariableType(CVarNode);
Result:=CTool.ExtractDefinitionType(CVarNode);
if Result='' then exit;
Result:=ConvertSimpleCTypeToPascalType(Result,true);
end;
@ -2379,14 +2408,14 @@ var
NeedH2PNode: Boolean;
begin
Result:=nil;
if (CNode.Desc=ccnVariable)
if (CNode.Desc=ccnDefinition)
and (CNode.FirstChild<>nil)
and (CNode.FirstChild.Desc=ccnUnion) then begin
// ToDo: union
end else begin
SubH2PNode:=nil;
if CNode.Desc=ccnVariable then
CCode:=CTool.ExtractVariableType(CNode)
if CNode.Desc=ccnDefinition then
CCode:=CTool.ExtractDefinitionType(CNode)
else if CNode.Desc=ccnFunction then
CCode:=CTool.ExtractFunctionResultType(CNode)
else if CNode.Desc=ccnFuncParameter then

View File

@ -200,14 +200,14 @@ begin
begin
inc(Position);
end;
#$EF:
if (Source[Position+1]=#$BB)
and (Source[Position+2]=#$BF) then begin
// skip UTF BOM
inc(Position,3);
end else begin
break;
end;
#$EF:
if (Source[Position+1]=#$BB)
and (Source[Position+2]=#$BF) then begin
// skip UTF BOM
inc(Position,3);
end else begin
break;
end;
'\': // backslash
if (Position<Len) and (Source[Position+1] in [#10,#13]) then begin
inc(Position,2);
@ -282,6 +282,13 @@ begin
if Position<=Len then begin
c1:=Source[Position];
case c1 of
#10,#13:
begin
inc(Position);
if (Position<=Len) and (Source[Position] in [#10,#13])
and (Source[Position]<>c1) then
inc(Position);
end;
'A'..'Z','a'..'z','_':
begin
// identifier
@ -352,8 +359,7 @@ begin
if Position<=Len then begin
c2:=Source[Position];
// test for double char operators
if ((c1=#13) and (c2=#10))
or ((c2='=') and (c1 in ['=','!','<','>','+','-','*','/','&','|']))
if ((c2='=') and (c1 in ['=','!','<','>','+','-','*','/','&','|']))
or ((c1=':') and (c2=':'))
or ((c1='|') and (c2='|'))
or ((c1='&') and (c2='&'))