paweld/fpc
1
0
Fork 0
mirror of https://gitlab.com/freepascal.org/fpc/source.git synced 2025-08-13 08:19:27 +02:00
Code Issues Packages Projects Releases Wiki Activity

fcl-js: started jsonscanner adaption for pas2js

Browse Source 
git-svn-id: trunk@40045 -
This commit is contained in:
Mattias Gaertner 2018-10-25 20:16:56 +00:00
parent 225aa064fe
commit 75e6356e02
2 changed files with 88 additions and 78 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

164
packages/fcl-json/src/jsonscanner.pp
View File

@ -15,6 +15,10 @@
{$mode objfpc} {$mode objfpc}
{$h+} {$h+}
{$ifdef fpc}
{$define UsePChar}
{$endif}
unit jsonscanner; unit jsonscanner;
interface interface
@ -48,7 +52,7 @@ type
tkUnknown tkUnknown
); );
EScannerError = class(EParserError); EScannerError = class(EParserError);
TJSONOption = (joUTF8,joStrict,joComments,joIgnoreTrailingComma); TJSONOption = (joUTF8,joStrict,joComments,joIgnoreTrailingComma);
TJSONOptions = set of TJSONOption; TJSONOptions = set of TJSONOption;
@ -62,24 +66,27 @@ Type
TJSONScanner = class TJSONScanner = class
private private
FSource : TStringList; FSource: TStringList;
FCurRow: Integer; FCurRow: Integer;
FCurToken: TJSONToken; FCurToken: TJSONToken;
FCurTokenString: string; FCurTokenString: string;
FCurLine: string; FCurLine: string;
TokenStr: PChar; FTokenStr: {$ifdef UsePChar}PChar{$else}integer{$endif}; // position inside FCurLine
FOptions : TJSONOptions; FOptions : TJSONOptions;
function GetCurColumn: Integer; inline; function GetCurColumn: Integer; inline;
function GetO(AIndex: TJSONOption): Boolean; function GetO(AIndex: TJSONOption): Boolean;
procedure SetO(AIndex: TJSONOption; AValue: Boolean); procedure SetO(AIndex: TJSONOption; AValue: Boolean);
protected protected
procedure Error(const Msg: string);overload; procedure Error(const Msg: string);overload;
procedure Error(const Msg: string; Const Args: array of Const);overload; procedure Error(const Msg: string;
Const Args: array of {$ifdef pas2js}jsvalue{$else}const{$endif});overload;
function DoFetchToken: TJSONToken; inline; function DoFetchToken: TJSONToken; inline;
public public
{$ifdef fpc}
constructor Create(Source : TStream; AUseUTF8 : Boolean = True); overload; deprecated 'use options form instead'; constructor Create(Source : TStream; AUseUTF8 : Boolean = True); overload; deprecated 'use options form instead';
constructor Create(const Source : String; AUseUTF8 : Boolean = True); overload; deprecated 'use options form instead'; constructor Create(const Source : String; AUseUTF8 : Boolean = True); overload; deprecated 'use options form instead';
constructor Create(Source: TStream; AOptions: TJSONOptions); overload; constructor Create(Source: TStream; AOptions: TJSONOptions); overload;
{$endif}
constructor Create(const Source: String; AOptions: TJSONOptions); overload; constructor Create(const Source: String; AOptions: TJSONOptions); overload;
destructor Destroy; override; destructor Destroy; override;
function FetchToken: TJSONToken; function FetchToken: TJSONToken;
@ -122,6 +129,7 @@ const
implementation implementation
{$ifdef fpc}
constructor TJSONScanner.Create(Source : TStream; AUseUTF8 : Boolean = True); constructor TJSONScanner.Create(Source : TStream; AUseUTF8 : Boolean = True);
Var Var
@ -155,6 +163,7 @@ begin
FSource.LoadFromStream(Source); FSource.LoadFromStream(Source);
FOptions:=AOptions; FOptions:=AOptions;
end; end;
{$endif}
constructor TJSONScanner.Create(const Source: String; AOptions: TJSONOptions); constructor TJSONScanner.Create(const Source: String; AOptions: TJSONOptions);
begin begin
@ -181,7 +190,8 @@ begin
raise EScannerError.Create(Msg); raise EScannerError.Create(Msg);
end; end;
procedure TJSONScanner.Error(const Msg: string; const Args: array of const); procedure TJSONScanner.Error(const Msg: string;
const Args: array of {$ifdef pas2js}jsvalue{$else}const{$endif});
begin begin
raise EScannerError.CreateFmt(Msg, Args); raise EScannerError.CreateFmt(Msg, Args);
end; end;
@ -194,13 +204,13 @@ function TJSONScanner.DoFetchToken: TJSONToken;
if Result then if Result then
begin begin
FCurLine:=FSource[FCurRow]; FCurLine:=FSource[FCurRow];
TokenStr:=PChar(FCurLine); FTokenStr:=PChar(FCurLine);
Inc(FCurRow); Inc(FCurRow);
end end
else else
begin begin
FCurLine:=''; FCurLine:='';
TokenStr:=nil; FTokenStr:=nil;
end; end;
end; end;
@ -214,7 +224,7 @@ var
IsStar,EOC: Boolean; IsStar,EOC: Boolean;
begin begin
if TokenStr = nil then if FTokenStr = nil then
if not FetchLine then if not FetchLine then
begin begin
Result := tkEOF; Result := tkEOF;
@ -224,7 +234,7 @@ begin
FCurTokenString := ''; FCurTokenString := '';
case TokenStr[0] of case FTokenStr[0] of
#0: // Empty line #0: // Empty line
begin begin
FetchLine; FetchLine;
@ -234,33 +244,33 @@ begin
begin begin
Result := tkWhitespace; Result := tkWhitespace;
repeat repeat
Inc(TokenStr); Inc(FTokenStr);
if TokenStr[0] = #0 then if FTokenStr[0] = #0 then
if not FetchLine then if not FetchLine then
begin begin
FCurToken := Result; FCurToken := Result;
exit; exit;
end; end;
until not (TokenStr[0] in [#9, ' ']); until not (FTokenStr[0] in [#9, ' ']);
end; end;
'"','''': '"','''':
begin begin
C:=TokenStr[0]; C:=FTokenStr[0];
If (C='''') and (joStrict in Options) then If (C='''') and (joStrict in Options) then
Error(SErrInvalidCharacter, [CurRow,CurColumn,TokenStr[0]]); Error(SErrInvalidCharacter, [CurRow,CurColumn,FTokenStr[0]]);
Inc(TokenStr); Inc(FTokenStr);
TokenStart := TokenStr; TokenStart := FTokenStr;
OldLength := 0; OldLength := 0;
FCurTokenString := ''; FCurTokenString := '';
while not (TokenStr[0] in [#0,C]) do while not (FTokenStr[0] in [#0,C]) do
begin begin
if (TokenStr[0]='\') then if (FTokenStr[0]='\') then
begin begin
// Save length // Save length
SectionLength := TokenStr - TokenStart; SectionLength := FTokenStr - TokenStart;
Inc(TokenStr); Inc(FTokenStr);
// Read escaped token // Read escaped token
Case TokenStr[0] of Case FTokenStr[0] of
'"' : S:='"'; '"' : S:='"';
'''' : S:=''''; '''' : S:='''';
't' : S:=#9; 't' : S:=#9;
@ -275,14 +285,14 @@ begin
u:=0; u:=0;
For I:=1 to 4 do For I:=1 to 4 do
begin begin
Inc(TokenStr); Inc(FTokenStr);
c2:=TokenStr^; c2:=FTokenStr^;
Case c2 of Case c2 of
'0'..'9': u:=u*16+ord(c2)-ord('0'); '0'..'9': u:=u*16+ord(c2)-ord('0');
'A'..'F': u:=u*16+ord(c2)-ord('A')+10; 'A'..'F': u:=u*16+ord(c2)-ord('A')+10;
'a'..'f': u:=u*16+ord(c2)-ord('a')+10; 'a'..'f': u:=u*16+ord(c2)-ord('a')+10;
else else
Error(SErrInvalidCharacter, [CurRow,CurColumn,TokenStr[0]]); Error(SErrInvalidCharacter, [CurRow,CurColumn,FTokenStr[0]]);
end; end;
end; end;
// ToDo: 4-bytes UTF16 // ToDo: 4-bytes UTF16
@ -293,7 +303,7 @@ begin
end; end;
#0 : Error(SErrOpenString); #0 : Error(SErrOpenString);
else else
Error(SErrInvalidCharacter, [CurRow,CurColumn,TokenStr[0]]); Error(SErrInvalidCharacter, [CurRow,CurColumn,FTokenStr[0]]);
end; end;
SetLength(FCurTokenString, OldLength + SectionLength+1+Length(S)); SetLength(FCurTokenString, OldLength + SectionLength+1+Length(S));
if SectionLength > 0 then if SectionLength > 0 then
@ -301,61 +311,61 @@ begin
Move(S[1],FCurTokenString[OldLength + SectionLength+1],Length(S)); Move(S[1],FCurTokenString[OldLength + SectionLength+1],Length(S));
Inc(OldLength, SectionLength+Length(S)); Inc(OldLength, SectionLength+Length(S));
// Next char // Next char
TokenStart := TokenStr+1; TokenStart := FTokenStr+1;
end; end;
if TokenStr[0] = #0 then if FTokenStr[0] = #0 then
Error(SErrOpenString); Error(SErrOpenString);
Inc(TokenStr); Inc(FTokenStr);
end; end;
if TokenStr[0] = #0 then if FTokenStr[0] = #0 then
Error(SErrOpenString); Error(SErrOpenString);
SectionLength := TokenStr - TokenStart; SectionLength := FTokenStr - TokenStart;
SetLength(FCurTokenString, OldLength + SectionLength); SetLength(FCurTokenString, OldLength + SectionLength);
if SectionLength > 0 then if SectionLength > 0 then
Move(TokenStart^, FCurTokenString[OldLength + 1], SectionLength); Move(TokenStart^, FCurTokenString[OldLength + 1], SectionLength);
Inc(TokenStr); Inc(FTokenStr);
Result := tkString; Result := tkString;
end; end;
',': ',':
begin begin
Inc(TokenStr); Inc(FTokenStr);
Result := tkComma; Result := tkComma;
end; end;
'0'..'9','.','-': '0'..'9','.','-':
begin begin
TokenStart := TokenStr; TokenStart := FTokenStr;
while true do while true do
begin begin
Inc(TokenStr); Inc(FTokenStr);
case TokenStr[0] of case FTokenStr[0] of
'.': '.':
begin begin
if TokenStr[1] in ['0'..'9', 'e', 'E'] then if FTokenStr[1] in ['0'..'9', 'e', 'E'] then
begin begin
Inc(TokenStr); Inc(FTokenStr);
repeat repeat
Inc(TokenStr); Inc(FTokenStr);
until not (TokenStr[0] in ['0'..'9', 'e', 'E','-','+']); until not (FTokenStr[0] in ['0'..'9', 'e', 'E','-','+']);
end; end;
break; break;
end; end;
'0'..'9': ; '0'..'9': ;
'e', 'E': 'e', 'E':
begin begin
Inc(TokenStr); Inc(FTokenStr);
if TokenStr[0] in ['-','+'] then if FTokenStr[0] in ['-','+'] then
Inc(TokenStr); Inc(FTokenStr);
while TokenStr[0] in ['0'..'9'] do while FTokenStr[0] in ['0'..'9'] do
Inc(TokenStr); Inc(FTokenStr);
break; break;
end; end;
else else
if not (TokenStr[0] in [#0,'}',']',',',#9,' ']) then if not (FTokenStr[0] in [#0,'}',']',',',#9,' ']) then
Error(SErrInvalidCharacter, [CurRow,CurColumn,TokenStr[0]]); Error(SErrInvalidCharacter, [CurRow,CurColumn,FTokenStr[0]]);
break; break;
end; end;
end; end;
SectionLength := TokenStr - TokenStart; SectionLength := FTokenStr - TokenStart;
FCurTokenString:=''; FCurTokenString:='';
SetString(FCurTokenString, TokenStart, SectionLength); SetString(FCurTokenString, TokenStart, SectionLength);
If (FCurTokenString[1]='.') then If (FCurTokenString[1]='.') then
@ -364,74 +374,74 @@ begin
end; end;
':': ':':
begin begin
Inc(TokenStr); Inc(FTokenStr);
Result := tkColon; Result := tkColon;
end; end;
'{': '{':
begin begin
Inc(TokenStr); Inc(FTokenStr);
Result := tkCurlyBraceOpen; Result := tkCurlyBraceOpen;
end; end;
'}': '}':
begin begin
Inc(TokenStr); Inc(FTokenStr);
Result := tkCurlyBraceClose; Result := tkCurlyBraceClose;
end; end;
'[': '[':
begin begin
Inc(TokenStr); Inc(FTokenStr);
Result := tkSquaredBraceOpen; Result := tkSquaredBraceOpen;
end; end;
']': ']':
begin begin
Inc(TokenStr); Inc(FTokenStr);
Result := tkSquaredBraceClose; Result := tkSquaredBraceClose;
end; end;
'/' : '/' :
begin begin
if Not (joComments in Options) then if Not (joComments in Options) then
Error(SErrInvalidCharacter, [CurRow,CurCOlumn,TokenStr[0]]); Error(SErrInvalidCharacter, [CurRow,CurCOlumn,FTokenStr[0]]);
TokenStart:=TokenStr; TokenStart:=FTokenStr;
Inc(TokenStr); Inc(FTokenStr);
Case Tokenstr[0] of Case FTokenStr[0] of
'/' : begin '/' : begin
SectionLength := Length(FCurLine)- (TokenStr - PChar(FCurLine)); SectionLength := Length(FCurLine)- (FTokenStr - PChar(FCurLine));
Inc(TokenStr); Inc(FTokenStr);
FCurTokenString:=''; FCurTokenString:='';
SetString(FCurTokenString, TokenStr, SectionLength); SetString(FCurTokenString, FTokenStr, SectionLength);
Fetchline; Fetchline;
end; end;
'*' : '*' :
begin begin
IsStar:=False; IsStar:=False;
Inc(TokenStr); Inc(FTokenStr);
TokenStart:=TokenStr; TokenStart:=FTokenStr;
Repeat Repeat
if (TokenStr[0]=#0) then if (FTokenStr[0]=#0) then
begin begin
SectionLength := (TokenStr - TokenStart); SectionLength := (FTokenStr - TokenStart);
S:=''; S:='';
SetString(S, TokenStart, SectionLength); SetString(S, TokenStart, SectionLength);
FCurtokenString:=FCurtokenString+S; FCurtokenString:=FCurtokenString+S;
if not fetchLine then if not fetchLine then
Error(SUnterminatedComment, [CurRow,CurCOlumn,TokenStr[0]]); Error(SUnterminatedComment, [CurRow,CurCOlumn,FTokenStr[0]]);
TokenStart:=TokenStr; TokenStart:=FTokenStr;
end; end;
IsStar:=TokenStr[0]='*'; IsStar:=FTokenStr[0]='*';
Inc(TokenStr); Inc(FTokenStr);
EOC:=(isStar and (TokenStr[0]='/')); EOC:=(isStar and (FTokenStr[0]='/'));
Until EOC; Until EOC;
if EOC then if EOC then
begin begin
SectionLength := (TokenStr - TokenStart-1); SectionLength := (FTokenStr - TokenStart-1);
S:=''; S:='';
SetString(S, TokenStart, SectionLength); SetString(S, TokenStart, SectionLength);
FCurtokenString:=FCurtokenString+S; FCurtokenString:=FCurtokenString+S;
Inc(TokenStr); Inc(FTokenStr);
end; end;
end; end;
else else
Error(SErrInvalidCharacter, [CurRow,CurCOlumn,TokenStr[0]]); Error(SErrInvalidCharacter, [CurRow,CurCOlumn,FTokenStr[0]]);
end; end;
Result:=tkComment; Result:=tkComment;
end; end;
@ -439,11 +449,11 @@ begin
begin begin
tstart:=CurRow; tstart:=CurRow;
Tcol:=CurColumn; Tcol:=CurColumn;
TokenStart := TokenStr; TokenStart := FTokenStr;
repeat repeat
Inc(TokenStr); Inc(FTokenStr);
until not (TokenStr[0] in ['A'..'Z', 'a'..'z', '0'..'9', '_']); until not (FTokenStr[0] in ['A'..'Z', 'a'..'z', '0'..'9', '_']);
SectionLength := TokenStr - TokenStart; SectionLength := FTokenStr - TokenStart;
FCurTokenString:=''; FCurTokenString:='';
SetString(FCurTokenString, TokenStart, SectionLength); SetString(FCurTokenString, TokenStart, SectionLength);
for it := tkTrue to tkNull do for it := tkTrue to tkNull do
@ -459,7 +469,7 @@ begin
Result:=tkIdentifier; Result:=tkIdentifier;
end; end;
else else
Error(SErrInvalidCharacter, [CurRow,CurCOlumn,TokenStr[0]]); Error(SErrInvalidCharacter, [CurRow,CurColumn,FTokenStr[0]]);
end; end;
FCurToken := Result; FCurToken := Result;
@ -467,7 +477,7 @@ end;
function TJSONScanner.GetCurColumn: Integer; function TJSONScanner.GetCurColumn: Integer;
begin begin
Result := TokenStr - PChar(CurLine); Result := FTokenStr - PChar(CurLine);
end; end;
function TJSONScanner.GetO(AIndex: TJSONOption): Boolean; function TJSONScanner.GetO(AIndex: TJSONOption): Boolean;

2
utils/pas2js/nodepas2js.pp
View File

@ -5,7 +5,7 @@ program nodepas2js;
uses uses
JS, NodeJS, NodeJSFS, Math, JS, NodeJS, NodeJSFS, Math,
Classes, SysUtils, Classes, SysUtils,
jsbase, jstree, jswriter, jsbase, jstree, jswriter, jsonscanner,// JSSrcMap,
fpjson, fpjson,
PasTree, PScanner, PParser, PasResolveEval, PasResolver, PasUseAnalyzer, PasTree, PScanner, PParser, PasResolveEval, PasResolver, PasUseAnalyzer,
FPPas2Js, FPPas2Js,