Skip to content

Commit

Permalink
Add support for big files (lager than 2GB)
Browse files Browse the repository at this point in the history 
The following allows installing RPM packages with size higher
than singed integer max value on 32-bit and 64-bit systems.

Signed-off-by: David Abdurachmanov <David.Abdurachmanov@cern.ch>
  • Loading branch information
David Abdurachmanov authored and David Abdurachmanov committed Oct 17, 2013
1 parent 937eda5 commit 0c12e66
Show file tree
Hide file tree
Showing 10 changed files with 45 additions and 45 deletions.
34 changes: 17 additions & 17 deletions apt-pkg/acquire-item.cc
View file
Expand Up @@ -46,7 +46,7 @@ using std::string;
// ---------------------------------------------------------------------
/* Returns false only if the checksums fail (the file not existing is not
a checksum mismatch) */
bool VerifyChecksums(string File,unsigned long Size,string MD5, string method)
bool VerifyChecksums(string File,unsigned long long Size,string MD5, string method)
{
struct stat Buf;

Expand All @@ -56,7 +56,7 @@ bool VerifyChecksums(string File,unsigned long Size,string MD5, string method)
// LORG:2006-03-09
// XXX hack alert: repomd doesn't have index sizes so ignore it and
// rely on checksum
if (Size > 0 && (unsigned long)Buf.st_size != Size)
if (Size > 0 && (unsigned long long)Buf.st_size != Size)
{
if (_config->FindB("Acquire::Verbose", false) == true)
cout << "Size of "<<File<<" did not match what's in the checksum list and was redownloaded."<<endl;
Expand Down Expand Up @@ -143,7 +143,7 @@ void pkgAcquire::Item::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
// ---------------------------------------------------------------------
/* Stash status and the file size. Note that setting Complete means
sub-phases of the acquire process such as decompresion are operating */
void pkgAcquire::Item::Start(string /*Message*/,unsigned long Size)
void pkgAcquire::Item::Start(string /*Message*/,unsigned long long Size)
{
Status = StatFetching;
if (FileSize == 0 && Complete == false)
Expand All @@ -153,15 +153,15 @@ void pkgAcquire::Item::Start(string /*Message*/,unsigned long Size)
// Acquire::Item::Done - Item downloaded OK /*{{{*/
// ---------------------------------------------------------------------
/* */
void pkgAcquire::Item::Done(string Message,off_t Size,string,
void pkgAcquire::Item::Done(string Message,unsigned long long Size,string,
pkgAcquire::MethodConfig *Cnf)
{
// We just downloaded something..
string FileName = LookupTag(Message,"Filename");
if (Complete == false && FileName == DestFile)
{
if (Owner->Log != 0)
Owner->Log->Fetched(Size,atoi(LookupTag(Message,"Resume-Point","0").c_str()));
Owner->Log->Fetched(Size,strtoull(LookupTag(Message,"Resume-Point","0").c_str(), NULL, 10));
}

if (FileSize == 0)
Expand Down Expand Up @@ -214,7 +214,7 @@ pkgAcqIndex::pkgAcqIndex(pkgAcquire *Owner,pkgRepository *Repository,
// If we're verifying authentication, check whether the size and
// checksums match, if not, delete the cached files and force redownload
string MD5Hash;
off_t Size;
unsigned long long Size;

if (Repository != NULL)
{
Expand Down Expand Up @@ -280,15 +280,15 @@ string pkgAcqIndex::Custom600Headers()
to the uncompressed version of the file. If this is so the file
is copied into the partial directory. In all other cases the file
is decompressed with a gzip uri. */
void pkgAcqIndex::Done(string Message,off_t Size,string MD5,
void pkgAcqIndex::Done(string Message,unsigned long long Size,string MD5,
pkgAcquire::MethodConfig *Cfg)
{
Item::Done(Message,Size,MD5,Cfg);

if (Decompression == true)
{
// CNC:2002-07-03
off_t FSize;
unsigned long long FSize;
string MD5Hash;

if (Repository != NULL && Repository->HasRelease() == true &&
Expand All @@ -305,7 +305,7 @@ void pkgAcqIndex::Done(string Message,off_t Size,string MD5,
ErrorText = _("Size mismatch");
Rename(DestFile,DestFile + ".FAILED");
if (_config->FindB("Acquire::Verbose",false) == true)
_error->Warning("Size mismatch of index file %s: %lu was supposed to be %lu",
_error->Warning("Size mismatch of index file %s: %llu was supposed to be %llu",
RealURI.c_str(), Size, FSize);
return;
}
Expand Down Expand Up @@ -423,7 +423,7 @@ pkgAcqIndexRel::pkgAcqIndexRel(pkgAcquire *Owner,pkgRepository *Repository,

// CNC:2002-07-09
string MD5Hash;
off_t Size;
unsigned long long Size;
if (Master == false && Repository != NULL)
{
if (Repository->HasRelease() == true)
Expand Down Expand Up @@ -485,7 +485,7 @@ string pkgAcqIndexRel::Custom600Headers()
/* The release file was not placed into the download directory then
a copy URI is generated and it is copied there otherwise the file
in the partial directory is moved into .. and the URI is finished. */
void pkgAcqIndexRel::Done(string Message,off_t Size,string MD5,
void pkgAcqIndexRel::Done(string Message,unsigned long long Size,string MD5,
pkgAcquire::MethodConfig *Cfg)
{
Item::Done(Message,Size,MD5,Cfg);
Expand Down Expand Up @@ -590,7 +590,7 @@ void pkgAcqIndexRel::Done(string Message,off_t Size,string MD5,
}

// CNC:2002-07-03
off_t FSize;
unsigned long long FSize;
string MD5Hash;
if (Master == false && Repository != NULL
&& Repository->HasRelease() == true
Expand All @@ -602,7 +602,7 @@ void pkgAcqIndexRel::Done(string Message,off_t Size,string MD5,
ErrorText = _("Size mismatch");
Rename(DestFile,DestFile + ".FAILED");
if (_config->FindB("Acquire::Verbose",false) == true)
_error->Warning("Size mismatch of index file %s: %lu was supposed to be %lu",
_error->Warning("Size mismatch of index file %s: %llu was supposed to be %llu",
RealURI.c_str(), Size, FSize);
return;
}
Expand Down Expand Up @@ -863,7 +863,7 @@ static void ScriptsAcquireDone(const char *ConfKey,
// AcqArchive::Done - Finished fetching /*{{{*/
// ---------------------------------------------------------------------
/* */
void pkgAcqArchive::Done(string Message,off_t Size,string Md5Hash,
void pkgAcqArchive::Done(string Message,unsigned long long Size,string Md5Hash,
pkgAcquire::MethodConfig *Cfg)
{
Item::Done(Message,Size,Md5Hash,Cfg);
Expand Down Expand Up @@ -984,7 +984,7 @@ void pkgAcqArchive::Finished()
// ---------------------------------------------------------------------
/* The file is added to the queue */
pkgAcqFile::pkgAcqFile(pkgAcquire *Owner,string URI,string MD5,
unsigned long Size,string Dsc,string ShortDesc) :
unsigned long long Size,string Dsc,string ShortDesc) :
Item(Owner), Md5Hash(MD5)
{
Retries = _config->FindI("Acquire::Retries",0);
Expand All @@ -1005,7 +1005,7 @@ pkgAcqFile::pkgAcqFile(pkgAcquire *Owner,string URI,string MD5,
if (stat(DestFile.c_str(),&Buf) == 0)
{
// Hmm, the partial file is too big, erase it
if ((unsigned)Buf.st_size > Size)
if ((unsigned long long)Buf.st_size > Size)
unlink(DestFile.c_str());
else
PartialSize = Buf.st_size;
Expand All @@ -1017,7 +1017,7 @@ pkgAcqFile::pkgAcqFile(pkgAcquire *Owner,string URI,string MD5,
// AcqFile::Done - Item downloaded OK /*{{{*/
// ---------------------------------------------------------------------
/* */
void pkgAcqFile::Done(string Message,off_t Size,string MD5,
void pkgAcqFile::Done(string Message,unsigned long long Size,string MD5,
pkgAcquire::MethodConfig *Cnf)
{
// Check the md5
Expand Down
18 changes: 9 additions & 9 deletions apt-pkg/acquire-item.h
View file
Expand Up @@ -45,8 +45,8 @@ class pkgAcquire::Item
* Do not use anonyomus enums, as this breaks SWIG in some cases */
enum StatusFlags {StatIdle, StatFetching, StatDone, StatError} Status;
string ErrorText;
unsigned long FileSize;
unsigned long PartialSize;
unsigned long long FileSize;
unsigned long long PartialSize;
const char *Mode;
unsigned long ID;
bool Complete;
Expand All @@ -60,9 +60,9 @@ class pkgAcquire::Item

// Action members invoked by the worker
virtual void Failed(string Message,pkgAcquire::MethodConfig *Cnf);
virtual void Done(string Message,off_t Size,string Md5Hash,
virtual void Done(string Message,unsigned long long Size,string Md5Hash,
pkgAcquire::MethodConfig *Cnf);
virtual void Start(string Message,unsigned long Size);
virtual void Start(string Message,unsigned long long Size);
virtual string Custom600Headers() {return string();}
virtual string DescURI() = 0;
virtual void Finished() {}
Expand Down Expand Up @@ -97,7 +97,7 @@ class pkgAcqIndex : public pkgAcquire::Item
public:

// Specialized action members
virtual void Done(string Message,off_t Size,string Md5Hash,
virtual void Done(string Message,unsigned long long Size,string Md5Hash,
pkgAcquire::MethodConfig *Cnf);
virtual string Custom600Headers();
virtual string DescURI() {return RealURI;} // CNC:2003-02-14
Expand Down Expand Up @@ -128,7 +128,7 @@ class pkgAcqIndexRel : public pkgAcquire::Item

// Specialized action members
virtual void Failed(string Message,pkgAcquire::MethodConfig *Cnf);
virtual void Done(string Message,off_t Size,string Md5Hash,
virtual void Done(string Message,unsigned long long Size,string Md5Hash,
pkgAcquire::MethodConfig *Cnf);
virtual string Custom600Headers();
virtual string DescURI() {return RealURI;}
Expand Down Expand Up @@ -161,7 +161,7 @@ class pkgAcqArchive : public pkgAcquire::Item

// Specialized action members
virtual void Failed(string Message,pkgAcquire::MethodConfig *Cnf);
virtual void Done(string Message,off_t Size,string Md5Hash,
virtual void Done(string Message,unsigned long long Size,string Md5Hash,
pkgAcquire::MethodConfig *Cnf);
virtual string MD5Sum() {return MD5;}
virtual string DescURI() {return Desc.URI;}
Expand All @@ -186,12 +186,12 @@ class pkgAcqFile : public pkgAcquire::Item

// Specialized action members
virtual void Failed(string Message,pkgAcquire::MethodConfig *Cnf);
virtual void Done(string Message,off_t Size,string Md5Hash,
virtual void Done(string Message,unsigned long long Size,string Md5Hash,
pkgAcquire::MethodConfig *Cnf);
virtual string MD5Sum() {return Md5Hash;}
virtual string DescURI() {return Desc.URI;}

pkgAcqFile(pkgAcquire *Owner,string URI,string MD5,unsigned long Size,
pkgAcqFile(pkgAcquire *Owner,string URI,string MD5,unsigned long long Size,
string Desc,string ShortDesc);
};

Expand Down
4 changes: 2 additions & 2 deletions apt-pkg/acquire-method.h
View file
Expand Up @@ -43,8 +43,8 @@ class pkgAcqMethod
time_t LastModified;
bool IMSHit;
string Filename;
unsigned long Size;
unsigned long ResumePoint;
unsigned long long Size;
unsigned long long ResumePoint;

void TakeHashes(Hashes &Hash);
FetchResult();
Expand Down
12 changes: 6 additions & 6 deletions apt-pkg/acquire-worker.cc
View file
Expand Up @@ -264,9 +264,9 @@ bool pkgAcquire::Worker::RunMessages()

CurrentItem = Itm;
CurrentSize = 0;
TotalSize = atoi(LookupTag(Message,"Size","0").c_str());
ResumePoint = atoi(LookupTag(Message,"Resume-Point","0").c_str());
Itm->Owner->Start(Message,atoi(LookupTag(Message,"Size","0").c_str()));
TotalSize = strtoull(LookupTag(Message,"Size","0").c_str(), NULL, 10);
ResumePoint = strtoull(LookupTag(Message,"Resume-Point","0").c_str(), NULL, 10);
Itm->Owner->Start(Message,strtoull(LookupTag(Message,"Size","0").c_str(), NULL, 10));

// Display update before completion
if (Log != 0 && Log->MorePulses == true)
Expand Down Expand Up @@ -296,13 +296,13 @@ bool pkgAcquire::Worker::RunMessages()

OwnerQ->ItemDone(Itm);
if (TotalSize != 0 &&
(unsigned)atoi(LookupTag(Message,"Size","0").c_str()) != TotalSize)
_error->Warning("Bizarre Error - File size is not what the server reported %s %lu",
strtoull(LookupTag(Message,"Size","0").c_str(), NULL, 10) != TotalSize)
_error->Warning("Bizarre Error - File size is not what the server reported %s %llu",
LookupTag(Message,"Size","0").c_str(),TotalSize);

// LORG:2006-03-09
// Look up the checksum type from owner
Owner->Done(Message,atoi(LookupTag(Message,"Size","0").c_str()),
Owner->Done(Message,strtoull(LookupTag(Message,"Size","0").c_str(), NULL, 10),
LookupTag(Message,Owner->ChecksumType().c_str()),Config);

ItemDone();
Expand Down
6 changes: 3 additions & 3 deletions apt-pkg/acquire-worker.h
View file
Expand Up @@ -68,9 +68,9 @@ class pkgAcquire::Worker
// The curent method state
pkgAcquire::Queue::QItem *CurrentItem;
string Status;
unsigned long CurrentSize;
unsigned long TotalSize;
unsigned long ResumePoint;
unsigned long long CurrentSize;
unsigned long long TotalSize;
unsigned long long ResumePoint;

// Load the method and do the startup
bool QueueItem(pkgAcquire::Queue::QItem *Item);
Expand Down
2 changes: 1 addition & 1 deletion apt-pkg/acquire.cc
View file
Expand Up @@ -866,7 +866,7 @@ void pkgAcquireStatus::Stop()
// AcquireStatus::Fetched - Called when a byte set has been fetched /*{{{*/
// ---------------------------------------------------------------------
/* This is used to get accurate final transfer rate reporting. */
void pkgAcquireStatus::Fetched(unsigned long Size,unsigned long Resume)
void pkgAcquireStatus::Fetched(unsigned long long Size,unsigned long long Resume)
{
FetchedBytes += Size - Resume;
}
Expand Down
4 changes: 2 additions & 2 deletions apt-pkg/acquire.h
View file
Expand Up @@ -176,7 +176,7 @@ class pkgAcquire::Queue

// Find a Queued item
QItem *FindItem(string URI,pkgAcquire::Worker *Owner);
bool ItemStart(QItem *Itm,unsigned long Size);
bool ItemStart(QItem *Itm,unsigned long long Size);
bool ItemDone(QItem *Itm);

bool Startup();
Expand Down Expand Up @@ -266,7 +266,7 @@ class pkgAcquireStatus
bool MorePulses;

// Called by items when they have finished a real download
virtual void Fetched(unsigned long Size,unsigned long ResumePoint);
virtual void Fetched(unsigned long long Size,unsigned long long ResumePoint);

// Called to change media
virtual bool MediaChange(string Media,string Drive) = 0;
Expand Down
4 changes: 2 additions & 2 deletions apt-pkg/repository.cc
View file
Expand Up @@ -69,7 +69,7 @@ bool pkgRepository::ParseRelease(string File)
File.c_str());

// Parse the size and append the directory
IndexChecksums[Path].Size = atoi(Size.c_str());
IndexChecksums[Path].Size = strtoull(Size.c_str(), NULL, 10);
IndexChecksums[Path].MD5 = Hash;
}

Expand All @@ -79,7 +79,7 @@ bool pkgRepository::ParseRelease(string File)
// Repository::FindChecksums - Get checksum info for file /*{{{*/
// ---------------------------------------------------------------------
/* */
bool pkgRepository::FindChecksums(string URI,off_t &Size, string &MD5)
bool pkgRepository::FindChecksums(string URI,unsigned long long &Size, string &MD5)
{
string Path = string(URI,RootURI.size());
if (IndexChecksums.find(Path) == IndexChecksums.end())
Expand Down
4 changes: 2 additions & 2 deletions apt-pkg/repository.h
View file
Expand Up @@ -16,7 +16,7 @@ class pkgRepository


struct Checksum {
unsigned long Size;
unsigned long long Size;
string MD5;
string SHA1;
};
Expand All @@ -40,7 +40,7 @@ class pkgRepository
virtual bool HasRelease() const { return GotRelease; }

virtual bool IsAuthenticated() const { return !FingerPrintList.empty(); }
virtual bool FindChecksums(string URI,off_t &Size, string &MD5);
virtual bool FindChecksums(string URI,unsigned long long &Size, string &MD5);
// Only used in repomd atm
virtual string FindURI(string DataType) {return "";}
// LORG:2006-02-23
Expand Down
2 changes: 1 addition & 1 deletion python/apt.i
View file
Expand Up @@ -368,7 +368,7 @@ class pkgRPyAcquireStatus : public pkgAcquireStatus
pkgAcquireStatus::CurrentItems;

/* Call only Python method, if existent, or parent method. */
void Fetched(unsigned long Size,unsigned long ResumePoint)
void Fetched(unsigned long long Size,unsigned long long ResumePoint)
{
PyObject *attr = PyObject_GetAttrString(PyObj, "Fetched");
if (attr != NULL) {
Expand Down

0 comments on commit 0c12e66

Please sign in to comment.