Skip to content
Permalink
Browse files

Stream package fetches directly to the local file.

Previously package downloads would go through download_file() and cache
the entire file in RAM before writing to disk.  Introduce a new
download_pkg() function which streams 4K at a time and significantly
reduces memory usage.
  • Loading branch information
jperkin committed Jul 31, 2015
1 parent e7efa1f commit 0e421ba4bbb144fd0be11db856618af2919824cf
Showing with 82 additions and 7 deletions.
  1. +6 −7 actions.c
  2. +75 −0 download.c
  3. +1 −0 pkgin.h
@@ -63,8 +63,8 @@ pkg_download(Plisthead *installhead)
FILE *fp;
Pkglist *pinstall;
struct stat st;
Dlfile *dlpkg;
char pkg_fs[BUFSIZ], pkg_url[BUFSIZ], query[BUFSIZ];
ssize_t size;
int rc = EXIT_SUCCESS;

printf(MSG_DOWNLOAD_PKGS);
@@ -109,7 +109,7 @@ pkg_download(Plisthead *installhead)
if ((fp = fopen(pkg_fs, "w")) == NULL)
err(EXIT_FAILURE, MSG_ERR_OPEN, pkg_fs);

if ((dlpkg = download_file(pkg_url, NULL)) == NULL) {
if ((size = download_pkg(pkg_url, fp)) == -1) {
fprintf(stderr, MSG_PKG_NOT_AVAIL, pinstall->depend);
rc = EXIT_FAILURE;

@@ -122,12 +122,11 @@ pkg_download(Plisthead *installhead)
continue;
}

fwrite(dlpkg->buf, dlpkg->size, 1, fp);
fclose(fp);

XFREE(dlpkg->buf);
XFREE(dlpkg);

if (size != pinstall->file_size) {
(void)unlink(pkg_fs);
errx(EXIT_FAILURE, "download mismatch: %s", pkg_fs);
}
} /* download loop */

return rc;
@@ -134,3 +134,78 @@ download_file(char *str_url, time_t *db_mtime)

return file;
}

/*
* Download a package to the local cache.
*/
ssize_t
download_pkg(char *pkg_url, FILE *fp)
{
struct url_stat st;
size_t size, wrote;
ssize_t fetched, written = 0;
off_t statsize = 0;
struct url *url;
fetchIO *f = NULL;
char buf[4096];
char *pkg, *ptr;

if ((url = fetchParseURL(pkg_url)) == NULL)
errx(EXIT_FAILURE, "%s: parse failure", pkg_url);

if ((f = fetchXGet(url, &st, "")) == NULL)
errx(EXIT_FAILURE, "%s: %s", pkg_url, fetchLastErrString);

/* Package not available */
if (st.size == -1)
return st.size;

if ((pkg = strrchr(pkg_url, '/')) != NULL)
pkg++;
else
pkg = (char *)pkg_url; /* should not happen */

if (parsable) {
printf(MSG_DOWNLOAD_START);
} else {
printf(MSG_DOWNLOADING, pkg);
fflush(stdout);
start_progress_meter(pkg, st.size, &statsize);
}

while (written < st.size) {
if ((fetched = fetchIO_read(f, buf, sizeof(buf))) == 0)
break;
if (fetched == -1 && errno == EINTR)
continue;
if (fetched == -1)
errx(EXIT_FAILURE, "fetch failure: %s",
fetchLastErrString);

statsize += fetched;
size = fetched;

for (ptr = buf; size > 0; ptr += wrote, size -= wrote) {
if ((wrote = fwrite(ptr, 1, size, fp)) < size) {
if (ferror(fp) && errno == EINTR)
clearerr(fp);
else
break;
}
written += wrote;
}
}

if (parsable)
printf(MSG_DOWNLOAD_END);
else
stop_progress_meter();

fetchIO_close(f);
fetchFreeURL(url);

if (written != st.size)
return -1;

return written;
}
@@ -228,6 +228,7 @@ extern Preflist **preflist;

/* download.c*/
Dlfile *download_file(char *, time_t *);
ssize_t download_pkg(char *, FILE *);
/* summary.c */
int update_db(int, char **);
void split_repos(void);

0 comments on commit 0e421ba

Please sign in to comment.
You can’t perform that action at this time.