LCL: added encoding UTF-8 with BOM, IDE: fixed changing encoding

git-svn-id: trunk@19211 -

ide/lazarusidestrconsts.pas

@@ -1707,6 +1707,7 @@ resourcestring
   lisAllYourModificationsToWillBeLostAndTheFileReopened = 'All your modificatio'
     +'ns to %s%s%s%swill be lost and the file reopened.';
   lisOpenLfm = 'Open %s';
+  lisUtf8WithBOM = 'UTF-8 with BOM';
   uemSetBookmark = '&Set Bookmark';
   uemToggleBookmark = '&Toggle Bookmark';
   uemReadOnly = 'Read Only';

ide/project.pp

@@ -1453,8 +1453,13 @@ end;
 
 function TUnitInfo.NeedsSaveToDisk: boolean;
 begin
-  Result:=IsVirtual or Modified or ChangedOnDisk(true)
+  Result:=IsVirtual or Modified or ChangedOnDisk(true);
-          or (not FileExistsUTF8(Filename));
+  if not Result then begin
+    if Source<>nil then
+      Result:=Source.FileOnDiskNeedsUpdate
+    else
+      Result:=not FileExistsUTF8(Filename);
+  end;
 end;
 
 procedure TUnitInfo.UpdateUsageCount(Min, IfBelowThis, IncIfBelow: extended);

ide/sourceeditor.pp

@@ -4023,6 +4023,8 @@ begin
     then begin
       // the ansi encoding is shown as 'ansi (system encoding)' -> cut
       NewEncoding:=EncodingAnsi;
+    end else if NewEncoding=lisUtf8WithBOM then begin
+      NewEncoding:=EncodingUTF8BOM;
     end;
     DebugLn(['TSourceNotebook.EncodingClicked NewEncoding=',NewEncoding]);
     if SrcEdit.CodeBuffer<>nil then begin
@@ -4055,7 +4057,7 @@ begin
         if CurResult=mrYes then begin
           // change file
           SrcEdit.CodeBuffer.DiskEncoding:=NewEncoding;
-          SrcEdit.Modified:=true;
+          SrcEdit.CodeBuffer.Modified:=true;
           DebugLn(['TSourceNotebook.EncodingClicked Change file to ',SrcEdit.CodeBuffer.DiskEncoding]);
           if (not SrcEdit.CodeBuffer.IsVirtual)
           and (LazarusIDE.DoSaveEditorFile(SrcEdit.PageIndex,[])<>mrOk) then begin
@@ -4453,13 +4455,16 @@ begin
   for i:=0 to List.Count-1 do begin
     CurName:='Encoding'+IntToStr(i);
     CurEncoding:=List[i];
+    CurCaption:=CurEncoding;
     if SysUtils.CompareText(CurEncoding,EncodingAnsi)=0 then begin
       SysEncoding:=GetSystemEncoding;
       if (SysEncoding<>'') and (SysUtils.CompareText(SysEncoding,EncodingAnsi)<>0)
       then
-        CurEncoding:=CurEncoding+' ('+GetSystemEncoding+')';
+        CurCaption:=CurCaption+' ('+GetSystemEncoding+')';
+    end;
+    if CurEncoding='UTF-8BOM' then begin
+      CurCaption:=lisUtf8WithBOM;
     end;
-    CurCaption:=CurEncoding;
     if SrcEditSubMenuEncoding.Count=i then begin
       // add new item
       IDEMenuItem:=RegisterIDEMenuCommand(SrcEditSubMenuEncoding,

lcl/lconvencoding.pas

@@ -32,6 +32,7 @@ uses
 const
   EncodingUTF8 = 'utf8';
   EncodingAnsi = 'ansi';
+  EncodingUTF8BOM = 'utf8bom';
 
 function GuessEncoding(const s: string): string;
 
@@ -48,6 +49,7 @@ var
   ConvertAnsiToUTF8: TConvertEncodingFunction = nil;
   ConvertUTF8ToAnsi: TConvertEncodingFunction = nil;
   
+function UTF8BOMToUTF8(const s: string): string; // UTF8 with BOM
 function ISO_8859_1ToUTF8(const s: string): string; // central europe
 function CP1250ToUTF8(const s: string): string; // central europe
 function CP1251ToUTF8(const s: string): string; // cyrillic
@@ -64,6 +66,7 @@ function KOI8ToUTF8(const s: string): string;  // russian cyrillic
 function SingleByteToUTF8(const s: string;
                           const Table: TCharToUTF8Table): string;
 
+function UTF8ToUTF8BOM(const s: string): string; // UTF8 with BOM
 function UTF8ToISO_8859_1(const s: string): string; // central europe
 function UTF8ToCP1250(const s: string): string; // central europe
 function UTF8ToCP1251(const s: string): string; // cyrillic
@@ -3524,6 +3527,11 @@ const
     ''                  // #255
   );
 
+function UTF8BOMToUTF8(const s: string): string;
+begin
+  Result:=copy(s,4,length(s));
+end;
+
 function ISO_8859_1ToUTF8(const s: string): string;
 begin
   Result:=SingleByteToUTF8(s,ArrayISO_8859_1ToUTF8);
@@ -4298,6 +4306,12 @@ begin
   end;
 end;
 
+function UTF8ToUTF8BOM(const s: string): string;
+begin
+  DebugLn(['UTF8ToUTF8BOM ']);
+  Result:=#$EF#$BB#$BF+s;
+end;
+
 function UTF8ToISO_8859_1(const s: string): string;
 begin
   Result:=UTF8ToSingleByte(s,@UnicodeToISO_8859_1);
@@ -4406,6 +4420,7 @@ end;
 procedure GetSupportedEncodings(List: TStrings);
 begin
   List.Add('UTF-8');
+  List.Add('UTF-8BOM');
   List.Add('Ansi');
   List.Add('CP1250');
   List.Add('CP1251');
@@ -4486,7 +4501,7 @@ begin
   
   // try BOM (Byte Order Mark)
   if CompareI(@s[1],#$EF#$BB#$BF,3) then begin
-    Result:=EncodingUTF8;
+    Result:=EncodingUTF8BOM;
     exit;
   end;
   
@@ -4554,6 +4569,7 @@ begin
   //DebugLn(['ConvertEncoding ',AFrom,' ',ATo]);
   
   if (AFrom=EncodingUTF8) then begin
+    if ATo='utf8bom' then begin Result:=UTF8ToUTF8BOM(s); exit; end;
     if ATo='iso88591' then begin Result:=UTF8ToISO_8859_1(s); exit; end;
     if ATo='cp1250' then begin Result:=UTF8ToCP1250(s); exit; end;
     if ATo='cp1251' then begin Result:=UTF8ToCP1251(s); exit; end;
@@ -4573,6 +4589,7 @@ begin
       exit;
     end;
   end else if ATo=EncodingUTF8 then begin
+    if AFrom='utf8bom' then begin Result:=UTF8BOMToUTF8(s); exit; end;
     if AFrom='iso88591' then begin Result:=ISO_8859_1ToUTF8(s); exit; end;
     if AFrom='cp1250' then begin Result:=CP1250ToUTF8(s); exit; end;
     if AFrom='cp1251' then begin Result:=CP1251ToUTF8(s); exit; end;