From bfd07c62336b2d76bad9cd1b1e6e7f74bfb5d940 Mon Sep 17 00:00:00 2001
From: michael <michael@freepascal.org>
Date: Mon, 9 Jul 2018 11:39:09 +0000
Subject: [PATCH] * Small demo showing how to tokenize a file into words

git-svn-id: trunk@39419 -
---
 .gitattributes                           |  2 +
 packages/regexpr/examples/splitwords.lpi | 57 ++++++++++++++++++++++++
 packages/regexpr/examples/splitwords.pp  | 30 +++++++++++++
 3 files changed, 89 insertions(+)
 create mode 100644 packages/regexpr/examples/splitwords.lpi
 create mode 100644 packages/regexpr/examples/splitwords.pp

diff --git a/.gitattributes b/.gitattributes
index 22839570d4..20faf5a166 100644
--- a/.gitattributes
+++ b/.gitattributes
@@ -7252,6 +7252,8 @@ packages/regexpr/Makefile.fpc svneol=native#text/plain
 packages/regexpr/Makefile.fpc.fpcmake svneol=native#text/plain
 packages/regexpr/examples/Makefile svneol=native#text/plain
 packages/regexpr/examples/Makefile.fpc svneol=native#text/plain
+packages/regexpr/examples/splitwords.lpi svneol=native#text/plain
+packages/regexpr/examples/splitwords.pp svneol=native#text/plain
 packages/regexpr/examples/testreg1.pp svneol=native#text/plain
 packages/regexpr/fpmake.pp svneol=native#text/plain
 packages/regexpr/src/old/regexpr.pp svneol=native#text/plain
diff --git a/packages/regexpr/examples/splitwords.lpi b/packages/regexpr/examples/splitwords.lpi
new file mode 100644
index 0000000000..cd6af5b8ae
--- /dev/null
+++ b/packages/regexpr/examples/splitwords.lpi
@@ -0,0 +1,57 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<CONFIG>
+  <ProjectOptions>
+    <Version Value="11"/>
+    <General>
+      <Flags>
+        <MainUnitHasCreateFormStatements Value="False"/>
+        <MainUnitHasTitleStatement Value="False"/>
+        <MainUnitHasScaledStatement Value="False"/>
+        <UseDefaultCompilerOptions Value="True"/>
+      </Flags>
+      <SessionStorage Value="InProjectDir"/>
+      <MainUnit Value="0"/>
+      <Title Value="Split words"/>
+      <UseAppBundle Value="False"/>
+      <ResourceType Value="res"/>
+    </General>
+    <BuildModes Count="1">
+      <Item1 Name="Default" Default="True"/>
+    </BuildModes>
+    <PublishOptions>
+      <Version Value="2"/>
+    </PublishOptions>
+    <RunParams>
+      <FormatVersion Value="2"/>
+      <Modes Count="0"/>
+    </RunParams>
+    <Units Count="1">
+      <Unit0>
+        <Filename Value="splitwords.pp"/>
+        <IsPartOfProject Value="True"/>
+      </Unit0>
+    </Units>
+  </ProjectOptions>
+  <CompilerOptions>
+    <Version Value="11"/>
+    <Target>
+      <Filename Value="splitwords"/>
+    </Target>
+    <SearchPaths>
+      <UnitOutputDirectory Value="lib/$(TargetCPU)-$(TargetOS)"/>
+    </SearchPaths>
+  </CompilerOptions>
+  <Debugging>
+    <Exceptions Count="3">
+      <Item1>
+        <Name Value="EAbort"/>
+      </Item1>
+      <Item2>
+        <Name Value="ECodetoolError"/>
+      </Item2>
+      <Item3>
+        <Name Value="EFOpenError"/>
+      </Item3>
+    </Exceptions>
+  </Debugging>
+</CONFIG>
diff --git a/packages/regexpr/examples/splitwords.pp b/packages/regexpr/examples/splitwords.pp
new file mode 100644
index 0000000000..9adf6cc657
--- /dev/null
+++ b/packages/regexpr/examples/splitwords.pp
@@ -0,0 +1,30 @@
+{$mode objfpc}
+{$H+}
+uses cwstring, sysutils, classes, uregexpr;
+
+Var
+  Split : TStringList;
+  S : String;
+  R : TRegexpr;  
+  E : TEncoding;
+
+begin
+  R:=nil;
+  Split:=TStringList.Create;
+  try
+    E:=TEncoding.UTF8;
+    Split.LoadFromFile(ParamStr(1),E);
+    S:=Split.Text;
+    r := TRegExpr.Create;
+    r.spaceChars:=r.spaceChars+'|&@#"''(§^!{})-[]*%`=+/.;:,?';
+    r.LineSeparators:=#10;
+    r.Expression :='(\b[^\d\s]+\b)';
+    if R.Exec(S) then
+      repeat 
+        Writeln('Found (pos: ',R.MatchPos[0],'): ',System.Copy (S, R.MatchPos [0], R.MatchLen[0]));
+      until not R.ExecNext;
+  finally 
+    r.Free;
+    split.free;
+  end;
+end.