diff --git a/packages/paszlib/readme.txt b/packages/paszlib/readme.txt index f9d80877e1..38bcc43b60 100644 --- a/packages/paszlib/readme.txt +++ b/packages/paszlib/readme.txt @@ -1,3 +1,99 @@ +Contents: +zipper.pp/TZipper +- Introduction +- Zip standards compliance +- Zip file format +- Zip64 support notes +paszlib +- Introduction +- Change Log +- File list +- Legal issues +- Archive Locations + +================= +zipper.pp/TZipper +================= + +Introduction +============ +Zipper.pp contains TZipper, an object-oriented wrapper for the paszlib units +that allows +- compressing/adding files/streams +- decompressing files/streams +- listing files +contained in a zip file. + +Zip standards compliance +======================== +TZipper is meant to help implement the most widely used and useful aspects of +the zip format, while following the official specifications +http://www.pkware.com/documents/casestudies/APPNOTE.TXT +(latest version reviewed for this readme: 6.3.3, September 1, 2012) +as much as possible. + +Not all (de)compression methods specified in the zip standard [1] are supported. +Encryption (either zip 2.0 or AES) is not supported, nor are multiple disk sets (spanning/splitting). +Please see the fpdoc help and the zipper.pp for details on using the class. + +Zip file format +=============== +The standard mentioned above documents the zip file format authoratively +and in detail. However, a brief summary can be useful: +A zip file consists of + +For each file: +local file header +(filename, uncompressed,compressed size etc) +optional extended file header +(e.g. zip64 extended info which overrides size above) +compressed file data + +Central directory: +- for each file: +central directory header +(much the same data as local file header+position of local file header) +optional extended file header (e.g. zip64 extended info which overrides the +above) + +if zip64 is used: one +zip64 end of central directory record +(mainly used to point to beginning of central directory) +zip64 end of central directory locator +(mainly used to point to zip64 end of central directory record) + +in any case: one +end of central directory record +(contains position of central directory, zip file comment etc) + +Zip64 support notes +=================== +The zip64 extensions that allow large files are supported: +- total zip file size and uncompressed sizes of >4Gb (up to FPC's limit of int64 + size for streams) +- > 65535 files per zip archive (up to FPC's limit of integer due to + collection.count) + +Write support: +zip64 headers are added after local file headers only if the uncompressed or +compressed sizes overflow the local file header space. This avoids wasting space. + +Each local zip64 file header variable overrides its corresponding variable in +the local file header only if it is not 0. If it is, the local version is used. + +Each central directory zip64 file header variable overrides its corresponding +variable in the central directory file header only if it is not 0. If it is, the +central directory file header version is used. + +If zip64 support is needed due to zip64 local/central file headers and/or the +number of files in the zip file, the zip64 alternatives to the end of central +diretory variables are always written. Although the zip standard doesn't seem to +require this explicitly, it doesn't forbid it either and other utilities such as +rar and Windows 7 built in zip support seem to require it. + +======= +paszlib +======= _____________________________________________________________________________ PASZLIB 1.0 May 11th, 1998 diff --git a/packages/paszlib/src/zinflate.pas b/packages/paszlib/src/zinflate.pas index d0239a1298..092a861117 100644 --- a/packages/paszlib/src/zinflate.pas +++ b/packages/paszlib/src/zinflate.pas @@ -3,7 +3,7 @@ unit zinflate; { inflate.c -- zlib interface to inflate modules Copyright (C) 1995-1998 Mark Adler - Pascal tranlastion + Pascal translation Copyright (C) 1998 by Jacques Nomssi Nzali For conditions of distribution and use, see copyright notice in readme.txt } diff --git a/packages/paszlib/src/zipper.pp b/packages/paszlib/src/zipper.pp index c0e912feb9..29e4382971 100644 --- a/packages/paszlib/src/zipper.pp +++ b/packages/paszlib/src/zipper.pp @@ -1,7 +1,7 @@ { - $Id: header,v 1.1 2000/07/13 06:33:45 michael Exp $ + $Id: header,v 1.3 2013/05/26 06:33:45 michael Exp $ This file is part of the Free Component Library (FCL) - Copyright (c) 1999-2000 by the Free Pascal development team + Copyright (c) 1999-2013 by the Free Pascal development team See the file COPYING.FPC, included in this distribution, for details about the copyright. @@ -26,15 +26,19 @@ Uses Const { Signatures } - END_OF_CENTRAL_DIR_SIGNATURE = $06054B50; - LOCAL_FILE_HEADER_SIGNATURE = $04034B50; - CENTRAL_FILE_HEADER_SIGNATURE = $02014B50; + END_OF_CENTRAL_DIR_SIGNATURE = $06054B50; + ZIP64_END_OF_CENTRAL_DIR_SIGNATURE = $06064B50; + ZIP64_END_OF_CENTRAL_DIR_LOCATOR_SIGNATURE = $07064B50; + LOCAL_FILE_HEADER_SIGNATURE = $04034B50; + CENTRAL_FILE_HEADER_SIGNATURE = $02014B50; + ZIP64_HEADER_ID = $0001; Type - Local_File_Header_Type = Packed Record + Local_File_Header_Type = Packed Record //1 per zipped file Signature : LongInt; //4 bytes - Extract_Version_Reqd : Word; - Bit_Flag : Word; + Extract_Version_Reqd : Word; //if zip64: >= 45 + {$warning TODO implement EFS/language enooding using UTF-8} + Bit_Flag : Word; //"General purpose bit flag in PKZip appnote Compress_Method : Word; Last_Mod_Time : Word; Last_Mod_Date : Word; @@ -42,16 +46,38 @@ Type Compressed_Size : LongWord; Uncompressed_Size : LongWord; Filename_Length : Word; - Extra_Field_Length : Word; + Extra_Field_Length : Word; //refers to Extensible data field size + end; + + Extensible_Data_Field_Header_Type = Packed Record + // Beginning of extra field + // after local file header + // after central directory header + Header_ID : Word; + //e.g. $0001 (ZIP64_HEADER_ID) Zip64 extended information extra field + //$0009 OS/2: extended attributes + //$000a NTFS: (Win32 really) + //$000d UNIX: uid, gid etc + Data_Size : Word; //size of following field data + //... field data should follow... + end; + + Zip64_Extended_Info_Field_Type = Packed Record //goes after Extensible_Data_Field_Header_Type + // overrides Local and Central Directory data + // stored in extra field + Original_Size : QWord; //Uncompressed file + Compressed_Size : QWord; //Compressed data + Relative_Hdr_Offset : QWord; //Offset that leads to local header record + Disk_Start_Number : LongWord; //on which disk this file starts end; { Define the Central Directory record types } Central_File_Header_Type = Packed Record Signature : LongInt; //4 bytes - MadeBy_Version : Word; - Extract_Version_Reqd : Word; - Bit_Flag : Word; + MadeBy_Version : Word; //if zip64: lower byte >= 45 + Extract_Version_Reqd : Word; //if zip64: >=45 + Bit_Flag : Word; //General purpose bit flag in PKZip appnote Compress_Method : Word; Last_Mod_Time : Word; Last_Mod_Date : Word; @@ -64,10 +90,11 @@ Type Starting_Disk_Num : Word; Internal_Attributes : Word; External_Attributes : LongWord; - Local_Header_Offset : LongWord; + Local_Header_Offset : LongWord; //todo: use zip64 and set to 0xFFFFFFFF if needed End; - End_of_Central_Dir_Type = Packed Record + End_of_Central_Dir_Type = Packed Record //End of central directory record + //1 per zip file, near end, before comment Signature : LongInt; //4 bytes Disk_Number : Word; Central_Dir_Start_Disk : Word; @@ -78,6 +105,26 @@ Type ZipFile_Comment_Length : Word; end; + Zip64_End_of_Central_Dir_type = Packed Record + Signature : LongInt; + Record_Size : QWord; + Version_Made_By : Word; //lower byte >= 45 + Extract_Version_Reqd : Word; //version >= 45 + Disk_Number : LongWord; + Central_Dir_Start_Disk : LongWord; + Entries_This_Disk : QWord; + Total_Entries : QWord; + Central_Dir_Size : QWord; + Start_Disk_Offset : QWord; + end; + + Zip64_End_of_Central_Dir_Locator_type = Packed Record //comes after Zip64_End_of_Central_Dir_type + Signature : LongInt; + Zip64_EOCD_Start_Disk : LongWord; //Starting disk for Zip64 End of Central Directory record + Central_Dir_Zip64_EOCD_Offset : QWord; //offset of Zip64 End of Central Directory record + Total_Disks : LongWord; //total number of disks (contained in zip) + end; + Const Crc_32_Tab : Array[0..255] of LongWord = ( $00000000, $77073096, $ee0e612c, $990951ba, $076dc419, $706af48f, $e963a535, $9e6495a3, @@ -264,13 +311,16 @@ Type FDateTime: TDateTime; FDiskFileName: String; FHeaderPos: int64; + FNeedsZip64: Boolean; //flags whether filesize is big enough so we need a zip64 entry FOS: Byte; - FSize: Integer; + FSize: Int64; FStream: TStream; FCompressionLevel: TCompressionlevel; function GetArchiveFileName: String; Protected + // For multi-disk support, a disk number property could be added here. Property HdrPos : int64 Read FHeaderPos Write FheaderPos; + Property NeedsZip64 : boolean Read FNeedsZip64 Write FNeedsZip64; Public constructor Create(ACollection: TCollection); override; function IsDirectory: Boolean; @@ -280,7 +330,7 @@ Type Published Property ArchiveFileName : String Read GetArchiveFileName Write FArchiveFileName; Property DiskFileName : String Read FDiskFileName Write FDiskFileName; - Property Size : Integer Read FSize Write FSize; + Property Size : Int64 Read FSize Write FSize; Property DateTime : TDateTime Read FDateTime Write FDateTime; property OS: Byte read FOS write FOS; property Attributes: LongInt read FAttributes write FAttributes; @@ -305,29 +355,32 @@ Type TZipper = Class(TObject) Private - FEntries: TZipFileEntries; - FZipping : Boolean; - FBufSize : LongWord; - FFileName : String; { Name of resulting Zip file } - FFileComment: String; - FFiles : TStrings; - FInMemSize : Integer; - FOutStream : TStream; - FInFile : TStream; { I/O file variables } - LocalHdr : Local_File_Header_Type; - CentralHdr : Central_File_Header_Type; - EndHdr : End_of_Central_Dir_Type; - FOnPercent : LongInt; - FOnProgress : TProgressEvent; - FOnEndOfFile : TOnEndOfFileEvent; - FOnStartFile : TOnStartFileEvent; + FEntries : TZipFileEntries; + FZipping : Boolean; + FBufSize : LongWord; + FFileName : String; { Name of resulting Zip file } + FFileComment : String; + FFiles : TStrings; + FInMemSize : Int64; + FZipFileNeedsZip64 : Boolean; //flags whether at least one file is big enough to require a zip64 record + FOutStream : TStream; + FInFile : TStream; { I/O file variables } + LocalHdr : Local_File_Header_Type; + LocalZip64ExtHdr: Extensible_Data_Field_Header_Type; //Extra field header fixed to zip64 (i.e. .ID=1) + LocalZip64Fld : Zip64_Extended_Info_Field_Type; //header is in LocalZip64ExtHdr + CentralHdr : Central_File_Header_Type; + EndHdr : End_of_Central_Dir_Type; + FOnPercent : LongInt; + FOnProgress : TProgressEvent; + FOnEndOfFile : TOnEndOfFileEvent; + FOnStartFile : TOnStartFileEvent; function CheckEntries: Integer; procedure SetEntries(const AValue: TZipFileEntries); Protected Procedure CloseInput(Item : TZipFileEntry); Procedure StartZipFile(Item : TZipFileEntry); Function UpdateZipHeader(Item : TZipFileEntry; FZip : TStream; ACRC : LongWord;AMethod : Word; AZipVersionReqd : Word; AZipBitFlag : Word) : Boolean; - Procedure BuildZipDirectory; + Procedure BuildZipDirectory; //Builds central directory based on local headers Procedure DoEndOfFile; Procedure ZipOneFile(Item : TZipFileEntry); virtual; Function OpenInput(Item : TZipFileEntry) : Boolean; @@ -335,6 +388,7 @@ Type Procedure SetBufSize(Value : LongWord); Procedure SetFileName(Value : String); Function CreateCompressor(Item : TZipFileEntry; AinFile,AZipStream : TStream) : TCompressor; virtual; + Property NeedsZip64 : boolean Read FZipFileNeedsZip64 Write FZipFileNeedsZip64; Public Constructor Create; Destructor Destroy;override; @@ -356,7 +410,7 @@ Type Property FileComment: String Read FFileComment Write FFileComment; // Deprecated. Use Entries.AddFileEntry(FileName) or Entries.AddFileEntries(List) instead. Property Files : TStrings Read FFiles; deprecated; - Property InMemSize : Integer Read FInMemSize Write FInMemSize; + Property InMemSize : Int64 Read FInMemSize Write FInMemSize; Property Entries : TZipFileEntries Read FEntries Write SetEntries; end; @@ -364,12 +418,12 @@ Type TFullZipFileEntry = Class(TZipFileEntry) private - FCompressedSize: LongWord; + FCompressedSize: QWord; FCompressMethod: Word; FCRC32: LongWord; Public Property CompressMethod : Word Read FCompressMethod; - Property CompressedSize : LongWord Read FCompressedSize; + Property CompressedSize : QWord Read FCompressedSize; property CRC32: LongWord read FCRC32 write FCRC32; end; @@ -402,9 +456,9 @@ Type FEntries : TFullZipFileEntries; FFiles : TStrings; FZipStream : TStream; { I/O file variables } - LocalHdr : Local_File_Header_Type; + LocalHdr : Local_File_Header_Type; //Local header, before compressed file data + LocalZip64Fld : Zip64_Extended_Info_Field_Type; //header is in LocalZip64ExtHdr CentralHdr : Central_File_Header_Type; - EndHdr : End_of_Central_Dir_Type; FOnPercent : LongInt; FOnProgress : TProgressEvent; @@ -414,6 +468,11 @@ Type Procedure OpenInput; Procedure CloseOutput(Item : TFullZipFileEntry; var OutStream: TStream); Procedure CloseInput; + Procedure FindEndHeaders( + out AEndHdr: End_of_Central_Dir_Type; + out AEndHdrPos: Int64; + out AEndZip64Hdr: Zip64_End_of_Central_Dir_type; + out AEndZip64HdrPos: Int64); Procedure ReadZipDirectory; Procedure ReadZipHeader(Item : TFullZipFileEntry; out AMethod : Word); Procedure DoEndOfFile; @@ -454,14 +513,18 @@ Type Implementation ResourceString - SErrBufsizeChange = 'Changing buffer size is not allowed while (un)zipping'; - SErrFileChange = 'Changing output file name is not allowed while (un)zipping'; - SErrInvalidCRC = 'Invalid CRC checksum while unzipping %s'; - SErrCorruptZIP = 'Corrupt ZIP file %s'; + SErrBufsizeChange = 'Changing buffer size is not allowed while (un)zipping.'; + SErrFileChange = 'Changing output file name is not allowed while (un)zipping.'; + SErrInvalidCRC = 'Invalid CRC checksum while unzipping %s.'; + SErrCorruptZIP = 'Corrupt ZIP file %s.'; SErrUnsupportedCompressionFormat = 'Unsupported compression format %d'; - SErrMissingFileName = 'Missing filename in entry %d'; - SErrMissingArchiveName = 'Missing archive filename in streamed entry %d'; + SErrUnsupportedMultipleDisksCD = 'A central directory split over multiple disks is unsupported.'; + SErrMaxEntries = 'Encountered %d file entries; maximum supported is %d.'; + SErrMissingFileName = 'Missing filename in entry %d.'; + SErrMissingArchiveName = 'Missing archive filename in streamed entry %d.'; SErrFileDoesNotExist = 'File "%s" does not exist.'; + SErrFileTooLarge = 'File size %d is larger than maximum supported size %d.'; + SErrPosTooLarge = 'Position/offset %d is larger than maximum supported %d.'; SErrNoFileName = 'No archive filename for examine operation.'; SErrNoStream = 'No stream is opened.'; @@ -488,6 +551,26 @@ begin end; end; +function SwapEDFH(const Values: Extensible_Data_Field_Header_Type): Extensible_Data_Field_Header_Type; +begin + with Values do + begin + Result.Header_ID := SwapEndian(Header_ID); + Result.Data_Size := SwapEndian(Data_Size); + end; +end; + +function SwapZ64EIF(const Values: Zip64_Extended_Info_Field_Type): Zip64_Extended_Info_Field_Type; +begin + with Values do + begin + Result.Original_Size := SwapEndian(Original_Size); + Result.Compressed_Size := SwapEndian(Compressed_Size); + Result.Relative_Hdr_Offset := SwapEndian(Relative_Hdr_Offset); + Result.Disk_Start_Number := SwapEndian(Disk_Start_Number); + end; +end; + function SwapCFH(const Values: Central_File_Header_Type): Central_File_Header_Type; begin with Values do @@ -526,6 +609,34 @@ begin Result.ZipFile_Comment_Length := SwapEndian(ZipFile_Comment_Length); end; end; + +function SwapZ64ECD(const Values: Zip64_End_of_Central_Dir_Type): Zip64_End_of_Central_Dir_Type; +begin + with Values do + begin + Result.Signature := SwapEndian(Signature); + Result.Record_Size := SwapEndian(Record_Size); + Result.Version_Made_By := SwapEndian(Version_Made_By); + Result.Extract_Version_Reqd := SwapEndian(Extract_Version_Reqd); + Result.Disk_Number := SwapEndian(Disk_Number); + Result.Central_Dir_Start_Disk := SwapEndian(Central_Dir_Start_Disk); + Result.Entries_This_Disk := SwapEndian(Entries_This_Disk); + Result.Total_Entries := SwapEndian(Total_Entries); + Result.Central_Dir_Size := SwapEndian(Central_Dir_Size); + Result.Start_Disk_Offset := SwapEndian(Start_Disk_Offset); + end; +end; + +function SwapZ64ECDL(const Values: Zip64_End_of_Central_Dir_Locator_type): Zip64_End_of_Central_Dir_Locator_type; +begin + with Values do + begin + Result.Signature := SwapEndian(Signature); + Result.Zip64_EOCD_Start_Disk := SwapEndian(Zip64_EOCD_Start_Disk); + Result.Central_Dir_Zip64_EOCD_Offset := SwapEndian(Central_Dir_Zip64_EOCD_Offset); + Result.Total_Disks := SwapEndian(Total_Disks); + end; +end; {$ENDIF FPC_BIG_ENDIAN} Procedure DateTimeToZipDateTime(DT : TDateTime; out ZD,ZT : Word); @@ -536,7 +647,21 @@ Var begin DecodeDate(DT,Y,M,D); DecodeTime(DT,H,N,S,MS); - Y:=Y-1980; + if Y<1980 then + begin + // Invalid date/time; set to earliest possible + Y:=0; + M:=1; + D:=1; + H:=0; + N:=0; + S:=0; + MS:=0; + end + else + begin + Y:=Y-1980; + end; ZD:=d+(32*M)+(512*Y); ZT:=(S div 2)+(32*N)+(2048*h); end; @@ -561,8 +686,12 @@ begin end; const - OS_FAT = 0; + OS_FAT = 0; //MS-DOS and OS/2 (FAT/VFAT/FAT32) OS_UNIX = 3; + OS_OS2 = 6; //OS/2 HPFS + OS_NTFS = 10; + OS_VFAT = 14; + OS_OSX = 19; UNIX_MASK = $F000; UNIX_FIFO = $1000; @@ -673,22 +802,22 @@ end; procedure TDeflater.Compress; - Var Buf : PByte; - I,Count,NewCount : Integer; + I,Count,NewCount : integer; C : TCompressionStream; - BytesNow : Integer; - NextMark : Integer; - OnBytes : Integer; - FSize : Integer; + BytesNow : Int64; + NextMark : Int64; + OnBytes : Int64; + FSize : Int64; begin CRC32Val:=$FFFFFFFF; Buf:=GetMem(FBufferSize); if FOnPercent = 0 then FOnPercent := 1; OnBytes:=Round((FInFile.Size * FOnPercent) / 100); - BytesNow:=0; NextMark := OnBytes; + BytesNow:=0; + NextMark := OnBytes; FSize:=FInfile.Size; Try C:=TCompressionStream.Create(FCompressionLevel,FOutFile,True); @@ -700,7 +829,7 @@ begin For I:=0 to Count-1 do UpdC32(Buf[i]); NewCount:=Count; - While (NewCount>0) do + while (NewCount>0) do NewCount:=NewCount-C.Write(Buf^,NewCount); inc(BytesNow,Count); if BytesNow>NextMark Then @@ -856,7 +985,7 @@ begin FirstCh:= TRUE; Crc32Val:=$FFFFFFFF; FOnBytes:=Round((FInFile.Size * FOnPercent) / 100); - While NOT InputEof do + While Not InputEof do begin Remaining:=Succ(MaxInBufIdx - InBufIdx); If Remaining>255 then @@ -871,7 +1000,7 @@ begin ProcessLine(OneString); end; end; - Crc32Val := NOT Crc32Val; + Crc32Val := Not Crc32Val; ProcessLine(''); end; @@ -994,7 +1123,7 @@ Var Begin CurrChild := CodeTable^[Parent].Child; { Find first Child that has descendants .. clear any that don't } - While (CurrChild <> -1) AND (CodeTable^[CurrChild].Child = -1) do + While (CurrChild <> -1) and (CodeTable^[CurrChild].Child = -1) do begin CodeTable^[Parent].Child := CodeTable^[CurrChild].Sibling; CodeTable^[CurrChild].Sibling := -1; @@ -1192,7 +1321,7 @@ Procedure TZipper.GetFileInfo; Var F : TZipFileEntry; Info : TSearchRec; - I : LongWord; + I : integer; //zip spec allows QWord but FEntries.Count does not support it {$IFDEF UNIX} UnixInfo: Stat; {$ENDIF} @@ -1272,123 +1401,310 @@ Procedure TZipper.StartZipFile(Item : TZipFileEntry); Begin FillChar(LocalHdr,SizeOf(LocalHdr),0); + FillChar(LocalZip64Fld,SizeOf(LocalZip64Fld),0); With LocalHdr do begin Signature := LOCAL_FILE_HEADER_SIGNATURE; - Extract_Version_Reqd := 10; + Extract_Version_Reqd := 10; //default value, v1.0 Bit_Flag := 0; Compress_Method := 1; DateTimeToZipDateTime(Item.DateTime,Last_Mod_Date,Last_Mod_Time); Crc32 := 0; Compressed_Size := 0; - Uncompressed_Size := Item.Size; + LocalZip64Fld.Compressed_Size := 0; + if Item.Size >= $FFFFFFFF then + begin + Uncompressed_Size := $FFFFFFFF; + LocalZip64Fld.Original_Size := Item.Size; + end + else + begin + Uncompressed_Size := Item.Size; + LocalZip64Fld.Original_Size := 0; + end; FileName_Length := 0; - Extra_Field_Length := 0; - end ; + if (LocalZip64Fld.Original_Size>0) or + (LocalZip64Fld.Compressed_Size>0) or + (LocalZip64Fld.Disk_Start_Number>0) or + (LocalZip64Fld.Relative_Hdr_Offset>0) then + Extra_Field_Length := SizeOf(LocalZip64ExtHdr) + SizeOf(LocalZip64Fld) + else + Extra_Field_Length := 0; + end; End; function TZipper.UpdateZipHeader(Item: TZipFileEntry; FZip: TStream; ACRC: LongWord; AMethod: Word; AZipVersionReqd: Word; AZipBitFlag: Word ): Boolean; + // Update header for a single zip file (local header) var - ZFileName : ShortString; + IsZip64 : boolean; //Must the local header be in zip64 format? + // Separate from zip64 status of entire zip file. + ZFileName : String; Begin - ZFileName:=Item.ArchiveFileName; + ZFileName := Item.ArchiveFileName; + IsZip64 := false; With LocalHdr do begin FileName_Length := Length(ZFileName); Crc32 := ACRC; - Result:=Not (Compressed_Size >= Uncompressed_Size); - If Not Result then - begin { No... } - Compress_Method := 0; { ...change stowage type } - Compressed_Size := Uncompressed_Size; { ...update compressed size } - end + if LocalZip64Fld.Original_Size > 0 then + Result := Not (FZip.Size >= LocalZip64Fld.Original_Size) else + Result := Not (Compressed_Size >= Uncompressed_Size); + if Item.CompressionLevel=clNone + then Result:=false; //user wishes override or invalid compression + If Not Result then begin - Compress_method:=AMethod; - Compressed_Size := FZip.Size; + Compress_Method := 0; // No use for compression: change storage type & compression size... + if LocalZip64Fld.Original_Size>0 then + begin + IsZip64 := true; + Compressed_Size := $FFFFFFFF; + LocalZip64Fld.Compressed_Size := LocalZip64Fld.Original_Size; + end + else + begin + Compressed_Size := Uncompressed_Size; + LocalZip64Fld.Compressed_Size := 0; + end; + end + else { Using compression } + begin + Compress_method := AMethod; Bit_Flag := Bit_Flag or AZipBitFlag; + if FZip.Size >= $FFFFFFFF then + begin + IsZip64 := true; + Compressed_Size := $FFFFFFFF; + LocalZip64Fld.Compressed_Size := FZip.Size; + end + else + begin + Compressed_Size := FZip.Size; + LocalZip64Fld.Compressed_Size := 0; + end; if AZipVersionReqd > Extract_Version_Reqd then Extract_Version_Reqd := AZipVersionReqd; end; + if (IsZip64) and (Extract_Version_Reqd<45) then + Extract_Version_Reqd := 45; end; + if IsZip64 then + LocalHdr.Extra_Field_Length:=SizeOf(LocalZip64ExtHdr)+SizeOf(LocalZip64Fld); FOutStream.WriteBuffer({$IFDEF ENDIAN_BIG}SwapLFH{$ENDIF}(LocalHdr),SizeOf(LocalHdr)); + // Append extensible field header+zip64 extensible field if needed: + if IsZip64 then + begin + FOutStream.WriteBuffer({$IFDEF ENDIAN_BIG}SwapEDFH{$ENDIF}(LocalZip64ExtHdr),SizeOf(LocalZip64ExtHdr)); + FOutStream.WriteBuffer({$IFDEF ENDIAN_BIG}SwapZ64EIF{$ENDIF}(LocalZip64Fld),SizeOf(LocalZip64Fld)); + end; FOutStream.WriteBuffer(ZFileName[1],Length(ZFileName)); End; Procedure TZipper.BuildZipDirectory; - +// Write out all central file headers using info from local headers Var - SavePos : int64; - HdrPos : int64; - CenDirPos : int64; - ACount : Word; - ZFileName : ShortString; - + SavePos : Int64; + HdrPos : Int64; //offset from disk where file begins to local header + CenDirPos : Int64; + ACount : QWord; //entry counter + ZFileName : string; //archive filename + IsZip64 : boolean; //local header=zip64 format? + MinReqdVersion: word; //minimum + ExtInfoHeader : Extensible_Data_Field_Header_Type; + Zip64ECD : Zip64_End_of_Central_Dir_type; + Zip64ECDL : Zip64_End_of_Central_Dir_Locator_type; Begin - ACount := 0; - CenDirPos := FOutStream.Position; - FOutStream.Seek(0,soBeginning); { Rewind output file } - HdrPos := FOutStream.Position; - FOutStream.ReadBuffer(LocalHdr, SizeOf(LocalHdr)); + ACount := 0; + CenDirPos := FOutStream.Position; + FOutStream.Seek(0,soBeginning); { Rewind output file } + HdrPos := FOutStream.Position; + FOutStream.ReadBuffer(LocalHdr, SizeOf(LocalHdr)); {$IFDEF FPC_BIG_ENDIAN} - LocalHdr := SwapLFH(LocalHdr); + LocalHdr := SwapLFH(LocalHdr); {$ENDIF} - Repeat - SetLength(ZFileName,LocalHdr.FileName_Length); - FOutStream.ReadBuffer(ZFileName[1], LocalHdr.FileName_Length); - SavePos := FOutStream.Position; - FillChar(CentralHdr,SizeOf(CentralHdr),0); - With CentralHdr do - begin - Signature := CENTRAL_FILE_HEADER_SIGNATURE; - MadeBy_Version := LocalHdr.Extract_Version_Reqd; - {$IFDEF UNIX} - MadeBy_Version := MadeBy_Version or (OS_UNIX shl 8); - {$ENDIF} - Move(LocalHdr.Extract_Version_Reqd, Extract_Version_Reqd, 26); - Last_Mod_Time:=localHdr.Last_Mod_Time; - Last_Mod_Date:=localHdr.Last_Mod_Date; - File_Comment_Length := 0; - Starting_Disk_Num := 0; - Internal_Attributes := 0; - {$IFDEF UNIX} - External_Attributes := Entries[ACount].Attributes shl 16; - {$ELSE} - External_Attributes := Entries[ACount].Attributes; - {$ENDIF} - Local_Header_Offset := HdrPos; - end; - FOutStream.Seek(0,soEnd); - FOutStream.WriteBuffer({$IFDEF FPC_BIG_ENDIAN}SwapCFH{$ENDIF}(CentralHdr),SizeOf(CentralHdr)); - FOutStream.WriteBuffer(ZFileName[1],Length(ZFileName)); - Inc(ACount); - FOutStream.Seek(SavePos + LocalHdr.Compressed_Size,soBeginning); - HdrPos:=FOutStream.Position; - FOutStream.ReadBuffer(LocalHdr, SizeOf(LocalHdr)); -{$IFDEF FPC_BIG_ENDIAN} - LocalHdr := SwapLFH(LocalHdr); -{$ENDIF} - Until LocalHdr.Signature = CENTRAL_FILE_HEADER_SIGNATURE; - FOutStream.Seek(0,soEnd); - FillChar(EndHdr,SizeOf(EndHdr),0); - With EndHdr do - begin - Signature := END_OF_CENTRAL_DIR_SIGNATURE; - Disk_Number := 0; - Central_Dir_Start_Disk := 0; - Entries_This_Disk := ACount; - Total_Entries := ACount; - Central_Dir_Size := FOutStream.Size-CenDirPos; - Start_Disk_Offset := CenDirPos; - ZipFile_Comment_Length := Length(FFileComment); - FOutStream.WriteBuffer({$IFDEF FPC_BIG_ENDIAN}SwapECD{$ENDIF}(EndHdr), SizeOf(EndHdr)); - if Length(FFileComment) > 0 then - FOutStream.WriteBuffer(FFileComment[1],Length(FFileComment)); - end; + Repeat + SetLength(ZFileName,LocalHdr.FileName_Length); + FOutStream.ReadBuffer(ZFileName[1], LocalHdr.FileName_Length); + IsZip64:=(LocalHdr.Compressed_Size=$FFFFFFFF) or (LocalHdr.Uncompressed_Size=$FFFFFFFF) or (HdrPos>=$FFFFFFFF); + FillChar(LocalZip64Fld,SizeOf(LocalZip64Fld),0); // easier to check compressed length + if LocalHdr.Extra_Field_Length>0 then + begin + SavePos := FOutStream.Position; + if (IsZip64 and (LocalHdr.Extra_Field_Length>=SizeOf(LocalZip64ExtHdr)+SizeOf(LocalZip64Fld))) then + while FOutStream.PositionMinReqdVersion then + MinReqdVersion:=Extract_Version_Reqd; + Last_Mod_Time:=localHdr.Last_Mod_Time; + Last_Mod_Date:=localHdr.Last_Mod_Date; + File_Comment_Length := 0; + Starting_Disk_Num := 0; + Internal_Attributes := 0; + {$IFDEF UNIX} + External_Attributes := Entries[ACount].Attributes shl 16; + {$ELSE} + External_Attributes := Entries[ACount].Attributes; + {$ENDIF} + if HdrPos>=$FFFFFFFF then + begin + FZipFileNeedsZip64:=true; + IsZip64:=true; + Local_Header_offset := $FFFFFFFF; + // LocalZip64Fld will be written out as central dir extra field later + LocalZip64Fld.Relative_Hdr_Offset := HdrPos; + end + else + Local_Header_Offset := HdrPos; + end; + FOutStream.Seek(0,soEnd); + FOutStream.WriteBuffer({$IFDEF FPC_BIG_ENDIAN}SwapCFH{$ENDIF}(CentralHdr),SizeOf(CentralHdr)); + FOutStream.WriteBuffer(ZFileName[1],Length(ZFileName)); + if IsZip64 then + begin + FOutStream.Seek(0,soEnd); + FOutStream.WriteBuffer({$IFDEF FPC_BIG_ENDIAN}SwapEDFH{$ENDIF}(LocalZip64ExtHdr),SizeOf(LocalZip64ExtHdr)); + FOutStream.WriteBuffer({$IFDEF FPC_BIG_ENDIAN}SwapZ64EIF{$ENDIF}(LocalZip64Fld),SizeOf(LocalZip64Fld)); + end; + + Inc(ACount); + // Move past compressed file data to next header: + if LocalHdr.Compressed_Size=$FFFFFFFF then + FOutStream.Seek(SavePos + LocalZip64Fld.Compressed_Size,soBeginning) + else + FOutStream.Seek(SavePos + LocalHdr.Compressed_Size,soBeginning); + HdrPos:=FOutStream.Position; + FOutStream.ReadBuffer(LocalHdr, SizeOf(LocalHdr)); + {$IFDEF FPC_BIG_ENDIAN} + LocalHdr := SwapLFH(LocalHdr); + {$ENDIF} + Until LocalHdr.Signature = CENTRAL_FILE_HEADER_SIGNATURE; + FOutStream.Seek(0,soEnd); + FillChar(EndHdr,SizeOf(EndHdr),0); + + // Write end of central directory record + // We'll use the zip64 variants to store counts etc + // and copy to the old record variables if possible + // This seems to match expected behaviour of unzippers like + // unrar that only look at the zip64 record + FillChar(Zip64ECD, SizeOf(Zip64ECD), 0); + Zip64ECD.Signature:=ZIP64_END_OF_CENTRAL_DIR_SIGNATURE; + FillChar(Zip64ECDL, SizeOf(Zip64ECDL), 0); + Zip64ECDL.Signature:=ZIP64_END_OF_CENTRAL_DIR_LOCATOR_SIGNATURE; + Zip64ECDL.Total_Disks:=1; //default and no support for multi disks yet anyway + With EndHdr do + begin + Signature := END_OF_CENTRAL_DIR_SIGNATURE; + Disk_Number := 0; + Central_Dir_Start_Disk := 0; + + Zip64ECD.Entries_This_Disk:=ACount; + Zip64ECD.Total_Entries:=Acount; + if ACount>$FFFF then + begin + FZipFileNeedsZip64 := true; + Entries_This_Disk := $FFFF; + Total_Entries := $FFFF; + end + else + begin + Entries_This_Disk := Zip64ECD.Entries_This_Disk; + Total_Entries := Zip64ECD.Total_Entries; + end; + + Zip64ECD.Central_Dir_Size := FOutStream.Size-CenDirPos; + if (Zip64ECD.Central_Dir_Size)>$FFFFFFFF then + begin + FZipFileNeedsZip64 := true; + Central_Dir_Size := $FFFFFFFF; + end + else + begin + Central_Dir_Size := Zip64ECD.Central_Dir_Size; + end; + + Zip64ECD.Start_Disk_Offset := CenDirPos; + if Zip64ECD.Start_Disk_Offset>$FFFFFFFF then + begin + FZipFileNeedsZip64 := true; + Start_Disk_Offset := $FFFFFFFF; + end + else + begin + Start_Disk_Offset := Zip64ECD.Start_Disk_Offset; + end; + + ZipFile_Comment_Length := Length(FFileComment); + + if FZipFileNeedsZip64 then + begin + //Write zip64 end of central directory record if needed + if MinReqdVersion<45 then + MinReqdVersion := 45; + Zip64ECD.Extract_Version_Reqd := MinReqdVersion; + Zip64ECD.Version_Made_By := MinReqdVersion; + Zip64ECD.Record_Size := SizeOf(Zip64ECD)-12; //Assumes no variable length field following + Zip64ECDL.Central_Dir_Zip64_EOCD_Offset := FOutStream.Position; + Zip64ECDL.Zip64_EOCD_Start_Disk := 0; + FOutStream.WriteBuffer({$IFDEF FPC_BIG_ENDIAN}SwapZ64ECD{$ENDIF}(Zip64ECD), SizeOf(Zip64ECD)); + + //Write zip64 end of central directory locator if needed + FOutStream.WriteBuffer({$IFDEF FPC_BIG_ENDIAN}SwapZ64ECDL{$ENDIF}(Zip64ECDL), SizeOf(Zip64ECDL)); + end; + + FOutStream.WriteBuffer({$IFDEF FPC_BIG_ENDIAN}SwapECD{$ENDIF}(EndHdr), SizeOf(EndHdr)); + if Length(FFileComment) > 0 then + FOutStream.WriteBuffer(FFileComment[1],Length(FFileComment)); + end; end; Function TZipper.CreateCompressor(Item : TZipFileEntry; AInFile,AZipStream : TStream) : TCompressor; @@ -1417,6 +1733,8 @@ Begin else begin TmpFileName:=ChangeFileExt(FFileName,'.tmp'); + if TmpFileName=FFileName then + TmpFileName:=TmpFileName+'.tmp'; ZipStream:=TFileStream.Create(TmpFileName,fmCreate); end; Try @@ -1453,8 +1771,7 @@ end; // Just like SaveToFile, but uses the FileName property Procedure TZipper.ZipAllFiles; - -Begin +begin SaveToFile(FileName); end; @@ -1472,29 +1789,25 @@ end; procedure TZipper.SaveToStream(AStream: TStream); Var - I : Integer; - filecnt : integer; + I : integer; //could be qword but limited by FEntries.Count begin FOutStream := AStream; If CheckEntries=0 then Exit; + FZipping:=True; Try - GetFileInfo; + GetFileInfo; //get info on file entries in zip - filecnt:=0; for I:=0 to FEntries.Count-1 do - begin ZipOneFile(FEntries[i]); - inc(filecnt); - end; - if filecnt>0 then + if FEntries.Count>0 then BuildZipDirectory; finally FZipping:=False; // Remove entries that have been added by CheckEntries from Files. - For I:=0 to FFiles.Count-1 do + for I:=0 to FFiles.Count-1 do FEntries.Delete(FEntries.Count-1); end; end; @@ -1548,7 +1861,9 @@ Var ComprPct : Double; begin - If (LocalHdr.Uncompressed_Size>0) then + if (FZipFileNeedsZip64) and (LocalZip64Fld.Original_Size>0) then + ComprPct := (100.0 * (LocalZip64Fld.Original_size - LocalZip64Fld.Compressed_Size)) / LocalZip64Fld.Original_Size + else if (LocalHdr.Uncompressed_Size>0) then ComprPct := (100.0 * (LocalHdr.Uncompressed_Size - LocalHdr.Compressed_Size)) / LocalHdr.Uncompressed_Size else ComprPct := 0; @@ -1564,16 +1879,38 @@ begin FFiles:=TStringList.Create; FEntries:=TZipFileEntries.Create(TZipFileEntry); FOnPercent:=1; + FZipFileNeedsZip64:=false; + LocalZip64ExtHdr.Header_ID:=ZIP64_HEADER_ID; + LocalZip64ExtHdr.Data_Size:=SizeOf(Zip64_Extended_Info_Field_Type); end; Function TZipper.CheckEntries : Integer; Var - I : Integer; + I : integer; //Could be QWord but limited by FFiles.Count begin - For I:=0 to FFiles.Count-1 do + for I:=0 to FFiles.Count-1 do FEntries.AddFileEntry(FFiles[i]); + + // Use zip64 when number of file entries + // or individual (un)compressed sizes + // require it. + if FEntries.Count >= $FFFF then + FZipFileNeedsZip64:=true; + + if not(FZipFileNeedsZip64) then + begin + for I:=0 to FFiles.Count-1 do + begin + if FEntries[i].FNeedsZip64 then + begin + FZipFileNeedsZip64:=true; + break; + end; + end; + end; + Result:=FEntries.Count; end; @@ -1614,14 +1951,14 @@ Var Path: String; OldDirectorySeparators: set of char; Begin - { the default RTL behaviour is broken on Unix platforms + { the default RTL behavior is broken on Unix platforms for Windows compatibility: it allows both '/' and '\' - as directory separator. We don't want that behaviour + as directory separator. We don't want that behavior here, since 'abc\' is a valid file name under Unix. - (mantis 15836) On the other hand, many archives on - windows have '/' as pathseparator, even Windows - generated .odt files. So we disable this for windows. + (mantis 15836) On the other hand, many archives on + Windows have '/' as pathseparator, even Windows + generated .odt files. So we disable this for Windows. } OldDirectorySeparators:=AllowDirectorySeparators; {$ifndef Windows} @@ -1633,7 +1970,7 @@ Begin FOnCreateStream(Self, OutStream, Item); // If FOnCreateStream didn't create one, we create one now. If (OutStream=Nil) then - Begin + begin if (Path<>'') then ForceDirectories(Path); AllowDirectorySeparators:=OldDirectorySeparators; @@ -1644,7 +1981,6 @@ Begin Result:=True; If Assigned(FOnStartFile) then FOnStartFile(Self,OutFileName); - End; @@ -1675,21 +2011,43 @@ Procedure TUnZipper.ReadZipHeader(Item : TFullZipFileEntry; out AMethod : Word); Var S : String; D : TDateTime; + ExtraFieldHdr: Extensible_Data_Field_Header_Type; + SavePos: int64; //could be qword but limited by stream Begin FZipStream.Seek(Item.HdrPos,soBeginning); FZipStream.ReadBuffer(LocalHdr,SizeOf(LocalHdr)); {$IFDEF FPC_BIG_ENDIAN} LocalHdr := SwapLFH(LocalHdr); {$ENDIF} + FillChar(LocalZip64Fld,SizeOf(LocalZip64Fld),0); //ensure no erroneous info With LocalHdr do begin SetLength(S,Filename_Length); FZipStream.ReadBuffer(S[1],Filename_Length); - //SetLength(E,Extra_Field_Length); - //FZipStream.ReadBuffer(E[1],Extra_Field_Length); - FZipStream.Seek(Extra_Field_Length,soCurrent); Item.ArchiveFileName:=S; Item.DiskFileName:=S; + SavePos:=FZipStream.Position; //after filename, before extra fields + if Extra_Field_Length>0 then + begin + SavePos := FZipStream.Position; + if (LocalHdr.Extra_Field_Length>=SizeOf(ExtraFieldHdr)+SizeOf(LocalZip64Fld)) then + while FZipStream.Position0 +// If valid zip64 end of directory found, AEndZip64HdrPos>0 var - Buf: PByte; - BufSize: Integer; - I: Integer; -begin - AZipFileComment := ''; - AEndHdrPos := AZip.Size - SizeOf(AEndHdr); - if AEndHdrPos < 0 then + EndZip64Locator: Zip64_End_of_Central_Dir_Locator_type; + procedure SearchForSignature; + // Search for end of central directory record signature + // If failed, set AEndHdrPos to 0 + var + I: Integer; + Buf: PByte; + BufSize: Integer; + result: boolean; begin - AEndHdrPos := -1; + result:=false; + // scan the last (64k + something) bytes for the END_OF_CENTRAL_DIR_SIGNATURE + // (zip file comments are 64k max). + BufSize := 65536 + SizeOf(AEndHdr) + 128; + if FZipStream.Size < BufSize then + BufSize := FZipStream.Size; + + Buf := GetMem(BufSize); + try + FZipStream.Seek(FZipStream.Size - BufSize, soBeginning); + FZipStream.ReadBuffer(Buf^, BufSize); + + for I := BufSize - SizeOf(AEndHdr) downto 0 do + begin + if (Buf[I] or (Buf[I + 1] shl 8) or (Buf[I + 2] shl 16) or (Buf[I + 3] shl 24)) = END_OF_CENTRAL_DIR_SIGNATURE then + begin + Move(Buf[I], AEndHdr, SizeOf(AEndHdr)); + {$IFDEF FPC_BIG_ENDIAN} + AEndHdr := SwapECD(AEndHdr); + {$ENDIF} + if (AEndHdr.Signature = END_OF_CENTRAL_DIR_SIGNATURE) and + (I + SizeOf(AEndHdr) + AEndHdr.ZipFile_Comment_Length = BufSize) then + begin + AEndHdrPos := FZipStream.Size - BufSize + I; + FZipStream.Seek(AEndHdrPos + SizeOf(AEndHdr), soBeginning); + SetLength(FFileComment, AEndHdr.ZipFile_Comment_Length); + FZipStream.ReadBuffer(FFileComment[1], Length(FFileComment)); + result:=true; //found it + break; + end; + end; + end; + finally + FreeMem(Buf); + end; + if not(result) then + begin + AEndHdrPos := 0; + FillChar(AEndHdr, SizeOf(AEndHdr), 0); + end; + end; + + procedure ZeroData; + begin + AEndHdrPos := 0; FillChar(AEndHdr, SizeOf(AEndHdr), 0); + AEndZip64HdrPos:=0; + FillChar(AEndZip64Hdr, SizeOf(AEndZip64Hdr), 0); + end; + +begin + // Zip64 records may not exist, so fill out default values + FillChar(AEndZip64Hdr,SizeOf(AEndZip64Hdr), 0); + AEndZip64HdrPos:=0; + // Look for end of central directory record from + // back of file based on signature (only way due to + // variable length zip comment etc) + FFileComment := ''; + // Zip file requires end of central dir header so + // is corrupt if it is smaller than that + if FZipStream.Size < SizeOf(AEndHdr) then + begin + ZeroData; exit; end; - AZip.Seek(AEndHdrPos, soBeginning); - AZip.ReadBuffer(AEndHdr, SizeOf(AEndHdr)); + + AEndHdrPos := FZipStream.Size - SizeOf(AEndHdr); + FZipStream.Seek(AEndHdrPos, soBeginning); + FZipStream.ReadBuffer(AEndHdr, SizeOf(AEndHdr)); {$IFDEF FPC_BIG_ENDIAN} AEndHdr := SwapECD(AEndHdr); {$ENDIF} - if (AEndHdr.Signature = END_OF_CENTRAL_DIR_SIGNATURE) and - (AEndHdr.ZipFile_Comment_Length = 0) then + // Search unless record is right at the end of the file: + if (AEndHdr.Signature <> END_OF_CENTRAL_DIR_SIGNATURE) or + (AEndHdr.ZipFile_Comment_Length <> 0) then + SearchForSignature; + if AEndHdrPos=0 then + begin + ZeroData; exit; + end; - // scan the last (64k + something) bytes for the END_OF_CENTRAL_DIR_SIGNATURE - // (zip file comments are 64k max) - BufSize := 65536 + SizeOf(AEndHdr) + 128; - if AZip.Size < BufSize then - BufSize := AZip.Size; - - Buf := GetMem(BufSize); - try - AZip.Seek(AZip.Size - BufSize, soBeginning); - AZip.ReadBuffer(Buf^, BufSize); - - for I := BufSize - SizeOf(AEndHdr) downto 0 do + // With a valid end of dir record, see if there's zip64 + // fields: + FZipStream.Seek(AEndHdrPos-SizeOf(Zip64_End_of_Central_Dir_Locator_type),soBeginning); + FZipStream.ReadBuffer(EndZip64Locator, SizeOf(EndZip64Locator)); + {$IFDEF FPC_BIG_ENDIAN} + EndZip64Locator := SwapZ64ECDL(EndZip64Locator); + {$ENDIF} + if EndZip64Locator.Signature=ZIP64_END_OF_CENTRAL_DIR_LOCATOR_SIGNATURE then + begin + //Read EndZip64Locator.Total_Disks when implementing multiple disks support + if EndZip64Locator.Central_Dir_Zip64_EOCD_Offset>High(Int64) then + raise EZipError.CreateFmt(SErrPosTooLarge,[EndZip64Locator.Central_Dir_Zip64_EOCD_Offset,High(Int64)]); + AEndZip64HdrPos:=EndZip64Locator.Central_Dir_Zip64_EOCD_Offset; + FZipStream.Seek(AEndZip64HdrPos, soBeginning); + FZipStream.ReadBuffer(AEndZip64Hdr, SizeOf(AEndZip64Hdr)); + {$IFDEF FPC_BIG_ENDIAN} + AEndZip64Hdr := SwapZ64ECD(AEndZip64Hdr); + {$ENDIF} + if AEndZip64Hdr.Signature<>ZIP64_END_OF_CENTRAL_DIR_SIGNATURE then begin - if (Buf[I] or (Buf[I + 1] shl 8) or (Buf[I + 2] shl 16) or (Buf[I + 3] shl 24)) = END_OF_CENTRAL_DIR_SIGNATURE then - begin - Move(Buf[I], AEndHdr, SizeOf(AEndHdr)); - {$IFDEF FPC_BIG_ENDIAN} - AEndHdr := SwapECD(AEndHdr); - {$ENDIF} - if (AEndHdr.Signature = END_OF_CENTRAL_DIR_SIGNATURE) and - (I + SizeOf(AEndHdr) + AEndHdr.ZipFile_Comment_Length = BufSize) then - begin - AEndHdrPos := AZip.Size - BufSize + I; - AZip.Seek(AEndHdrPos + SizeOf(AEndHdr), soBeginning); - SetLength(AZipFileComment, AEndHdr.ZipFile_Comment_Length); - AZip.ReadBuffer(AZipFileComment[1], Length(AZipFileComment)); - exit; - end; - end; + //Corrupt header + ZeroData; + Exit; end; - - AEndHdrPos := -1; - FillChar(AEndHdr, SizeOf(AEndHdr), 0); - finally - FreeMem(Buf); + end + else + begin + // No zip64 data, so follow the offset in the end of central directory record + AEndZip64HdrPos:=0; + FillChar(AEndZip64Hdr, SizeOf(AEndZip64Hdr), 0); end; end; Procedure TUnZipper.ReadZipDirectory; Var - i : LongWord; //todo: expand to 8 bytes when introducing zip64 format + EndHdr : End_of_Central_Dir_Type; + EndZip64Hdr : Zip64_End_of_Central_Dir_type; + i : integer; //could be Qword but limited to number of items in collection EndHdrPos, - CenDirPos : Int64; + EndZip64HdrPos, + CenDirPos, + SavePos : Int64; //could be QWord but limited to stream maximums + ExtraFieldHeader : Extensible_Data_Field_Header_Type; + EntriesThisDisk : QWord; + Zip64Field: Zip64_Extended_Info_Field_Type; NewNode : TFullZipFileEntry; D : TDateTime; S : String; Begin - FindEndHeader(FZipStream, EndHdr, EndHdrPos, FFileComment); - if EndHdrPos < 0 then + FindEndHeaders(EndHdr, EndHdrPos, + EndZip64Hdr, EndZip64HdrPos); + if EndHdrPos=0 then raise EZipError.CreateFmt(SErrCorruptZIP,[FileName]); - CenDirPos := EndHdr.Start_Disk_Offset; + if (EndZip64HdrPos>0) and (EndZip64Hdr.Start_Disk_Offset>0) then + begin + if EndZip64Hdr.Start_Disk_Offset>High(Int64) then + raise EZipError.CreateFmt(SErrPosTooLarge,[EndZip64Hdr.Start_Disk_Offset,High(Int64)]); + CenDirPos := EndZip64Hdr.Start_Disk_Offset; + end + else + CenDirPos := EndHdr.Start_Disk_Offset; FZipStream.Seek(CenDirPos,soBeginning); FEntries.Clear; - for i:=0 to EndHdr.Entries_This_Disk-1 do + if (EndZip64HdrPos>0) and (EndZip64Hdr.Entries_This_Disk>0) then + begin + EntriesThisDisk := EndZip64Hdr.Entries_This_Disk; + if EntriesThisDisk<>EndZip64Hdr.Total_Entries then + raise EZipError.Create(SErrUnsupportedMultipleDisksCD); + end + else + begin + EntriesThisDisk :=EndHdr.Entries_This_Disk; + if EntriesThisDisk<>EndHdr.Total_Entries then + raise EZipError.Create(SErrUnsupportedMultipleDisksCD); + end; + + // Entries are added to a collection. The max number of items + // in a collection limits the entries we can process. + if EntriesThisDisk>MaxInt then + raise EZipError.CreateFmt(SErrMaxEntries,[EntriesThisDisk,MaxInt]); + + // Using while instead of for loop so qword can be used on 32 bit as well. + for i:=0 to EntriesThisDisk-1 do begin FZipStream.ReadBuffer(CentralHdr, SizeOf(CentralHdr)); {$IFDEF FPC_BIG_ENDIAN} @@ -1787,24 +2256,62 @@ Begin if Signature<>CENTRAL_FILE_HEADER_SIGNATURE then raise EZipError.CreateFmt(SErrCorruptZIP,[FileName]); NewNode:=FEntries.Add as TFullZipFileEntry; + // Header position will be corrected later with zip64 version, if needed.. NewNode.HdrPos := Local_Header_Offset; SetLength(S,Filename_Length); FZipStream.ReadBuffer(S[1],Filename_Length); + SavePos:=FZipStream.Position; //After fixed part of central directory... + // and the filename; before any extra field(s) NewNode.ArchiveFileName:=S; + // Size/compressed size will be adjusted by zip64 entries if needed... NewNode.Size:=Uncompressed_Size; NewNode.FCompressedSize:=Compressed_Size; NewNode.CRC32:=CRC32; NewNode.OS := MadeBy_Version shr 8; - if NewNode.OS = OS_UNIX then NewNode.Attributes := External_Attributes shr 16 else NewNode.Attributes := External_Attributes; ZipDateTimeToDateTime(Last_Mod_Date,Last_Mod_Time,D); NewNode.DateTime:=D; - FZipStream.Seek(Extra_Field_Length+File_Comment_Length,soCurrent); + + // Go through any extra fields and extract any zip64 info + if Extra_Field_Length>0 then + begin + while (FZipStream.Position 0 then + NewNode.FCompressedSize := Zip64Field.Compressed_Size; + if Zip64Field.Original_Size>0 then + NewNode.Size := Zip64Field.Original_Size; + if Zip64Field.Relative_Hdr_Offset<>0 then + begin + if Zip64Field.Relative_Hdr_Offset>High(Int64) then + raise EZipError.CreateFmt(SErrPosTooLarge,[Zip64Field.Relative_Hdr_Offset,High(Int64)]); + NewNode.HdrPos := Zip64Field.Relative_Hdr_Offset; + end; + end + else + begin + // Read past non-Zip64 extra field + FZipStream.Seek(ExtraFieldHeader.Data_Size,soFromCurrent); + end; + end; + end; + // Move past extra fields and file comment to next header + FZipStream.Seek(SavePos+Extra_Field_Length+File_Comment_Length,soFromBeginning); end; - end; + end; end; Function TUnZipper.CreateDeCompressor(Item : TZipFileEntry; AMethod : Word;AZipFile,AOutFile : TStream) : TDeCompressor; @@ -1829,14 +2336,16 @@ Var IsLink: Boolean; IsCustomStream: Boolean; - procedure DoUnzip(const Dest: TStream); begin if ZMethod=0 then begin if (LocalHdr.Compressed_Size<>0) then begin - Count:=Dest.CopyFrom(FZipStream,LocalHdr.Compressed_Size) + if LocalZip64Fld.Compressed_Size>0 then + Count:=Dest.CopyFrom(FZipStream,LocalZip64Fld.Compressed_Size) + else + Count:=Dest.CopyFrom(FZipStream,LocalHdr.Compressed_Size); {$warning TODO: Implement CRC Check} end else @@ -1860,7 +2369,6 @@ Begin IsCustomStream := Assigned(FOnCreateStream); - if (IsCustomStream = False) and (FOutputPath<>'') then OutputFileName:=IncludeTrailingPathDelimiter(FOutputPath)+OutputFileName; @@ -1874,7 +2382,6 @@ Begin end; {$ENDIF} - if IsCustomStream then begin try @@ -1915,7 +2422,6 @@ Begin end; end; - if Not IsCustomStream then begin // set attributes @@ -1925,12 +2431,12 @@ Begin begin Attrs := 0; {$IFDEF UNIX} - if Item.OS = OS_UNIX then Attrs := Item.Attributes; - if Item.OS = OS_FAT then + if (Item.OS in [OS_UNIX,OS_OSX]) then Attrs := Item.Attributes; + if (Item.OS in [OS_FAT,OS_NTFS,OS_OS2,OS_VFAT]) then Attrs := ZipFatAttrsToUnixAttrs(Item.Attributes); {$ELSE} - if Item.OS = OS_FAT then Attrs := Item.Attributes; - if Item.OS = OS_UNIX then + if (Item.OS in [OS_FAT,OS_NTFS,OS_OS2,OS_VFAT]) then Attrs := Item.Attributes; + if (Item.OS in [OS_UNIX,OS_OSX]) then Attrs := ZipUnixAttrsToFatAttrs(ExtractFileName(Item.ArchiveFileName), Item.Attributes); {$ENDIF} @@ -1949,9 +2455,9 @@ end; Procedure TUnZipper.UnZipAllFiles; Var - Item : TFullZipFileEntry; - I : Integer; - AllFiles : Boolean; + Item : TFullZipFileEntry; + I : integer; //Really QWord but limited to FEntries.Count + AllFiles : Boolean; Begin FUnZipping:=True; @@ -1960,7 +2466,7 @@ Begin OpenInput; Try ReadZipDirectory; - For I:=0 to FEntries.Count-1 do + for i:=0 to FEntries.Count-1 do begin Item:=FEntries[i]; if AllFiles or (FFiles.IndexOf(Item.ArchiveFileName)<>-1) then @@ -2023,10 +2529,24 @@ Procedure TUnZipper.DoEndOfFile; Var ComprPct : Double; - + Uncompressed: QWord; + Compressed: QWord; begin - If (LocalHdr.Uncompressed_Size>0) then - ComprPct := (100.0 * (LocalHdr.Uncompressed_Size - LocalHdr.Compressed_Size)) / LocalHdr.Uncompressed_Size + If LocalZip64Fld.Original_Size > 0 then + Uncompressed := LocalZip64Fld.Original_Size + else + Uncompressed := LocalHdr.Uncompressed_Size; + + If LocalZip64Fld.Compressed_Size > 0 then + Compressed := LocalZip64Fld.Compressed_Size + else + Compressed := LocalHdr.Compressed_Size; + + If (Compressed>0) and (Uncompressed>0) then + if (Compressed>Uncompressed) then + ComprPct := (-100.0 * (Compressed - Uncompressed)) / Uncompressed + else + ComprPct := (100.0 * (Uncompressed - Compressed)) / Uncompressed else ComprPct := 0; If Assigned(FOnEndOfFile) then @@ -2091,6 +2611,8 @@ begin FOS := OS_FAT; {$ENDIF} FCompressionLevel:=cldefault; + FDateTime:=now; + FNeedsZip64:=false; inherited create(ACollection); end; @@ -2181,6 +2703,7 @@ begin For I:=0 to List.Count-1 do AddFileEntry(List[i]); end; + { TFullZipFileEntries } function TFullZipFileEntries.GetFZ(AIndex : Integer): TFullZipFileEntry; diff --git a/packages/paszlib/tests/tczipper.pp b/packages/paszlib/tests/tczipper.pp index 487e009fab..f5b2e5a785 100644 --- a/packages/paszlib/tests/tczipper.pp +++ b/packages/paszlib/tests/tczipper.pp @@ -1,50 +1,114 @@ program tczipper; { This file is part of the Free Pascal packages. - Copyright (c) 1999-2012 by the Free Pascal development team + Copyright (c) 2012-2013 by the Free Pascal Development Team + Created by Reinier Olislagers Tests zip/unzip functionality provided by the FPC zipper.pp unit. + If passed a zip file name as first argument, it will try and decompress + and list the contents of the zip file. See the file COPYING.FPC, included in this distribution, - for details about the copyright. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + for details about the license. **********************************************************************} {$mode objfpc}{$h+} -uses SysUtils, classes, zipper, md5; +//Define this if you want to inspect the generated zips etc +{$define KEEPTESTFILES} + +uses SysUtils, classes, zipper, unzip, zdeflate, zinflate, zip, md5, zstream, nullstream; type - TCallBackHandler = class(TObject) + + { TCallBackHandler } + + TCallBackHandler = class(TObject) //Callbacks used in zip/unzip processing + private + FPerformChecks: boolean; + FOriginalContent: string; + FShowContent: boolean; + FStreamResult: boolean; public + property PerformChecks: boolean read FPerformChecks write FPerformChecks; //If false, do not perform any consistency checks + property OriginalContent: string read FOriginalContent write FOriginalContent; //Zip entry uncompressed content used in TestZipEntries + property ShowContent: boolean read FShowContent write FShowContent; //Show contents of zip when extracting? + property StreamResult: boolean read FStreamResult; //For handler to report success/failure procedure EndOfFile(Sender:TObject; const Ratio:double); procedure StartOfFile(Sender:TObject; const AFileName:string); + procedure DoCreateZipOutputStream(Sender: TObject; var AStream: TStream; + AItem: TFullZipFileEntry); + procedure DoDoneOutZipStream(Sender: TObject; var AStream: TStream; + AItem: TFullZipFileEntry); //Used to verify zip entry decompressed contents + constructor Create; end; - -procedure TCallBackHandler.EndOfFile(Sender : TObject; Const Ratio : Double); +procedure TCallBackHandler.EndOfFile(Sender: TObject; const Ratio: double); begin - if (Ratio<0) then + writeln('End of file handler hit; ratio: '+floattostr(ratio)); + if (FPerformChecks) and (Ratio<0) then begin writeln('Found compression ratio '+floattostr(Ratio)+', which should never be lower than 0.'); - halt(3); + halt(1); end; end; -procedure TCallBackHandler.StartOfFile(Sender : TObject; Const AFileName : String); +procedure TCallBackHandler.StartOfFile(Sender: TObject; const AFileName: string); begin - if AFileName='' then + writeln('Start of file handler hit; filename: '+AFileName); + if (FPerformChecks) and (AFileName='') then begin writeln('Archive filename should not be empty.'); - halt(4); + halt(1); end; end; +procedure TCallBackHandler.DoCreateZipOutputStream(Sender: TObject; var AStream: TStream; + AItem: TFullZipFileEntry); +begin + AStream:=TMemoryStream.Create; +end; + +procedure TCallBackHandler.DoDoneOutZipStream(Sender: TObject; var AStream: TStream; + AItem: TFullZipFileEntry); +var + DecompressedContent: string; +begin + //writeln('At end of '+AItem.ArchiveFileName); + AStream.Position:=0; + SetLength(DecompressedContent,Astream.Size); + if AStream.Size>0 then + (AStream as TMemoryStream).Read(DecompressedContent[1], AStream.Size); + if (FPerformChecks) and (DecompressedContent<>OriginalContent) then + begin + FStreamResult:=false; + writeln('TestZipEntries failed: found entry '+AItem.ArchiveFileName+ + ' has value '); + writeln('*'+DecompressedContent+'*'); + writeln('expected '); + writeln('*'+OriginalContent+'*'); + end; + if (FPerformChecks=false) and (ShowContent=true) then + begin + //display only + writeln('TestZipEntries info: found entry '+AItem.ArchiveFileName+ + ' has value '); + writeln('*'+DecompressedContent+'*'); + end; + Astream.Free; +end; + +constructor TCallBackHandler.Create; +begin + FOriginalContent:='A'; //nice short demo content + FStreamResult:=true; + FPerformChecks:=true; //perform verification by default + FShowContent:=true; +end; + + +function CompareCompressDecompress: boolean; var - code: cardinal; CallBackHandler: TCallBackHandler; CompressedFile: string; FileContents: TStringList; @@ -55,10 +119,10 @@ var OurZipper: TZipper; UnZipper: TUnZipper; begin - code := 0; + result:=true; UncompressedFile1:=SysUtils.GetTempFileName('', 'UNC'); UncompressedFile2:=SysUtils.GetTempFileName('', 'UNC'); - CompressedFile:=SysUtils.GetTempFileName('', 'ZP'); + CompressedFile:=SysUtils.GetTempFileName('', 'CC'); FileContents:=TStringList.Create; OurZipper:=TZipper.Create; @@ -93,8 +157,10 @@ begin end; // Delete original files + {$IFNDEF KEEPTESTFILES} DeleteFile(UncompressedFile1); DeleteFile(UncompressedFile2); + {$ENDIF} // Now unzip Unzipper.FileName:=CompressedFile; @@ -109,7 +175,7 @@ begin (not FileExists(UncompressedFile2)) then begin writeln('Unzip failed: could not find decompressed files.'); - halt(6); + exit(false); end; // Compare hashes @@ -120,25 +186,510 @@ begin then begin writeln('Unzip failed: uncompressed files are not the same as the originals.'); - halt(7); + exit(false); end; - if code = 0 then - writeln('Basic zip/unzip tests passed') - else - writeln('Basic zip/unzip test failed: ', code); finally FileContents.Free; CallBackHandler.Free; OurZipper.Free; UnZipper.Free; + {$IFNDEF KEEPTESTFILES} try if FileExists(CompressedFile) then DeleteFile(CompressedFile); if FileExists(UncompressedFile1) then DeleteFile(UncompressedFile1); if FileExists(UncompressedFile2) then DeleteFile(UncompressedFile2); finally - // Ignore errors; operating system should clean out temp files - end; + // Ignore errors: OS should eventually clean out temp files anyway + end; + {$ENDIF} end; +end; + +function CompressSmallStreams: boolean; +// Compresses some small streams using default compression and +// no compression (storage) +// Just storing is the best option; compression will enlarge the zip. +// Test verifies that the entries in the zip are not bigger than +// the originals. +var + DestFile: string; + z: TZipper; + zfe: TZipFileEntry; + s: string = 'abcd'; + DefaultStream, StoreStream: TStringStream; +begin + result:=true; + DestFile:=SysUtils.GetTempFileName('', 'CS1'); + z:=TZipper.Create; + z.FileName:=DestFile; + try + DefaultStream:=TStringStream.Create(s); + StoreStream:=TStringStream.Create(s); + + //DefaultStream - compression level = Default + zfe:=z.Entries.AddFileEntry(DefaultStream, 'Compressed'); + z.ZipAllFiles; + + if (z.Entries[0].Size>zfe.Size) then + begin + result:=false; + writeln('Small stream test default compression failed: compressed size '+ + inttostr(z.Entries[0].Size) + ' > original size '+inttostr(zfe.Size)); + exit; + end; + + finally + DefaultStream.Free; + StoreStream.Free; + z.Free; + end; + + {$IFNDEF KEEPTESTFILES} + try + DeleteFile(DestFile); + except + // ignore mess + end; + {$ENDIF} + + DestFile:=SysUtils.GetTempFileName('', 'CS2'); + z:=TZipper.Create; + z.FileName:=DestFile; + try + DefaultStream:=TStringStream.Create(s); + StoreStream:=TStringStream.Create(s); + + //StoreStream - compression level = Store + zfe:=z.Entries.AddFileEntry(StoreStream, 'Uncompressed'); + zfe.CompressionLevel:=clnone; + z.ZipAllFiles; + + if (z.Entries[0].Size>zfe.Size) then + begin + result:=false; + writeln('Small stream test uncompressed failed: compressed size '+ + inttostr(z.Entries[0].Size) + ' > original size '+inttostr(zfe.Size)); + exit; + end; + finally + DefaultStream.Free; + StoreStream.Free; + z.Free; + end; + + {$IFNDEF KEEPTESTFILES} + try + DeleteFile(DestFile); + except + // ignore mess + end; + {$ENDIF} + + //The result can be checked with the command (on Linux): + //unzip -v + //The column Size Shows that compressed files are bigger than source files +end; + +function ShowZipFile(ZipFile: string): boolean; +// Reads zip file and lists entries +var + CallBackHandler: TCallBackHandler; + i: integer; + UnZipper: TUnZipper; + UnzipArchiveFiles: TStringList; +begin + result:=true; + UnZipper:=TUnZipper.Create; + CallBackHandler:=TCallBackHandler.Create; + UnzipArchiveFiles:=TStringList.Create; + try + CallBackHandler.PerformChecks:=false; //only display output + UnZipper.FileName:=ZipFile; + Unzipper.Examine; + writeln('ShowZipFile: zip file has '+inttostr(UnZipper.Entries.Count)+' entries'); + + i:=0; + Unzipper.OnCreateStream:=@CallBackHandler.DoCreateZipOutputStream; + Unzipper.OnDoneStream:=@CallBackHandler.DoDoneOutZipStream; + while iEntries) then + begin + result:=false; + writeln('TestZipEntries failed: found '+ + inttostr(UnZipper.Entries.Count) + ' entries; expected '+inttostr(Entries)); + exit; + end; + i:=0; + Unzipper.OnCreateStream:=@CallBackHandler.DoCreateZipOutputStream; + Unzipper.OnDoneStream:=@CallBackHandler.DoDoneOutZipStream; + while iEntries) then + begin + result:=false; + writeln('TestEmptyZipEntries failed: found '+ + inttostr(UnZipper.Entries.Count) + ' entries; expected '+inttostr(Entries)); + exit; + end; + i:=0; + while iArchiveFile) then + begin + result:=false; + writeln('TestLargeFileName failed: found filename length '+ + inttostr(Length(Unzipper.Entries[0].ArchiveFileName))); + writeln('*'+Unzipper.Entries[0].ArchiveFileName + '*'); + writeln('Expected length '+inttostr(Length(ArchiveFile))); + writeln('*'+ArchiveFile+'*'); + exit; + end; + finally + Unzipper.Free; + end; + + {$IFNDEF KEEPTESTFILES} + try + DeleteFile(DestFile); + except + // ignore mess + end; + {$ENDIF} +end; + +function TestLargeZip64: boolean; +// Tests single zip file with large uncompressed content +// which forces it to zip64 format +var + ArchiveFile: string; + Buffer: PChar; + DestFile: string; + ContentStream: TNullStream; //empty contents + UnZipper: TUnZipper; + Zipper: TZipper; + i: int64; +begin + result:=true; + DestFile:=SysUtils.GetTempFileName('', 'LZ'); + Zipper:=TZipper.Create; + Zipper.FileName:=DestFile; + ArchiveFile:='HugeString.txt'; + + ContentStream:=TNullStream.Create; + // About 4Gb; content of 4 bytes+1 added + ContentStream.Size:=(1+$FFFFFFFF); + ContentStream.Position:=0; + writeln('Buffer created'); + try + Zipper.Entries.AddFileEntry(ContentStream, ArchiveFile); + writeln('entry added'); + Zipper.ZipAllFiles; + finally + ContentStream.Free; + Zipper.Free; + end; + + UnZipper:=TUnZipper.Create; + try + UnZipper.FileName:=DestFile; + Unzipper.Examine; + if (UnZipper.Entries.Count<>1) then + begin + result:=false; + writeln('TestLargeZip64 failed: found '+ + inttostr(UnZipper.Entries.Count) + ' entries; expected 1'); + exit; + end; + if (Unzipper.Entries[0].ArchiveFileName<>ArchiveFile) then + begin + result:=false; + writeln('TestLargeZip64 failed: found filename length '+ + inttostr(Length(Unzipper.Entries[0].ArchiveFileName))); + writeln('*'+Unzipper.Entries[0].ArchiveFileName + '*'); + writeln('Expected length '+inttostr(Length(ArchiveFile))); + writeln('*'+ArchiveFile+'*'); + exit; + end; + finally + Unzipper.Free; + end; + + {$IFNDEF KEEPTESTFILES} + try + DeleteFile(DestFile); + except + // ignore mess + end; + {$ENDIF} +end; + +var + code: cardinal; //test result code: 0 for success +begin + code:=0; + try + if FileExists(ParamStr(1)) then + begin + writeln(''); + writeln('Started investigating file '+ParamStr(1)); + ShowZipFile(ParamStr(1)); + writeln('Finished investigating file '+ParamStr(1)); + writeln(''); + end; + + writeln('CompareCompressDecompress started'); + if not(CompareCompressDecompress) then code:=code+2; //1 already taken by callback handler + writeln('CompareCompressDecompress finished'); + writeln(''); + writeln('CompressSmallStreams started'); + if not(CompressSmallStreams) then code:=code+4; + writeln('CompressSmallStreams finished'); + writeln(''); + writeln('TestZipEntries(2) started'); + if not(TestZipEntries(2)) then code:=code+8; + writeln('TestZipEntries(2) finished'); + writeln(''); + writeln('TestLargeFileName started'); + if not(TestLargeFileName) then code:=code+16; + writeln('TestLargeFileName finished'); + writeln(''); + writeln('TestEmptyZipEntries(10) started'); + // Run testemptyzipentries with a small number to test the test itself... as + // well as zip structure generated with empty files. + if not(TestEmptyZipEntries(10)) then code:=code+32; + writeln('TestEmptyZipEntries(10) finished'); + writeln(''); + writeln('TestEmptyZipEntries(65537) started'); + writeln('(note: this will take a long time)'); + {Note: tested tools with this file: + - info-zip unzip 6.0 + - Ionic's DotNetZip library unzip.exe utility verison 1.9.1.8 works + - 7zip's 7za 9.22 beta works. + } + if not(TestEmptyZipEntries(65537)) then code:=code+32; + writeln('TestEmptyZipEntries(65537) finished'); + writeln(''); + { This test will take a very long time as it tries to zip a 4Gb memory block. + It is therefore commented out by default } + { + writeln('TestLargeZip64 - started'); + if not(TestLargeZip64) then code:=code+thefollowingstatuscode; + writeln('TestLargeZip64 format - finished'); + writeln(''); + } + except + on E: Exception do + begin + writeln(''); + writeln('Exception: '); + writeln(E.Message); + writeln(''); + end; + end; + + if code=0 then + writeln('Basic zip/unzip tests passed: code '+inttostr(code)) + else + writeln('Basic zip/unzip tests failed: code '+inttostr(code)); Halt(code); end.