Permalink
Browse files

Stream package fetches directly to the local file.

Previously package downloads would go through download_file() and cache
the entire file in RAM before writing to disk.  Introduce a new
download_pkg() function which streams 4K at a time and significantly
reduces memory usage.
  • Loading branch information...
jperkin committed Jul 13, 2015
1 parent e7efa1f commit 0e421ba4bbb144fd0be11db856618af2919824cf
Showing with 82 additions and 7 deletions.
  1. +6 −7 actions.c
  2. +75 −0 download.c
  3. +1 −0 pkgin.h
View
@@ -63,8 +63,8 @@ pkg_download(Plisthead *installhead)
FILE *fp;
Pkglist *pinstall;
struct stat st;
- Dlfile *dlpkg;
char pkg_fs[BUFSIZ], pkg_url[BUFSIZ], query[BUFSIZ];
+ ssize_t size;
int rc = EXIT_SUCCESS;
printf(MSG_DOWNLOAD_PKGS);
@@ -109,7 +109,7 @@ pkg_download(Plisthead *installhead)
if ((fp = fopen(pkg_fs, "w")) == NULL)
err(EXIT_FAILURE, MSG_ERR_OPEN, pkg_fs);
- if ((dlpkg = download_file(pkg_url, NULL)) == NULL) {
+ if ((size = download_pkg(pkg_url, fp)) == -1) {
fprintf(stderr, MSG_PKG_NOT_AVAIL, pinstall->depend);
rc = EXIT_FAILURE;
@@ -122,12 +122,11 @@ pkg_download(Plisthead *installhead)
continue;
}
- fwrite(dlpkg->buf, dlpkg->size, 1, fp);
fclose(fp);
-
- XFREE(dlpkg->buf);
- XFREE(dlpkg);
-
+ if (size != pinstall->file_size) {
+ (void)unlink(pkg_fs);
+ errx(EXIT_FAILURE, "download mismatch: %s", pkg_fs);
+ }
} /* download loop */
return rc;
View
@@ -134,3 +134,78 @@ download_file(char *str_url, time_t *db_mtime)
return file;
}
+
+/*
+ * Download a package to the local cache.
+ */
+ssize_t
+download_pkg(char *pkg_url, FILE *fp)
+{
+ struct url_stat st;
+ size_t size, wrote;
+ ssize_t fetched, written = 0;
+ off_t statsize = 0;
+ struct url *url;
+ fetchIO *f = NULL;
+ char buf[4096];
+ char *pkg, *ptr;
+
+ if ((url = fetchParseURL(pkg_url)) == NULL)
+ errx(EXIT_FAILURE, "%s: parse failure", pkg_url);
+
+ if ((f = fetchXGet(url, &st, "")) == NULL)
+ errx(EXIT_FAILURE, "%s: %s", pkg_url, fetchLastErrString);
+
+ /* Package not available */
+ if (st.size == -1)
+ return st.size;
+
+ if ((pkg = strrchr(pkg_url, '/')) != NULL)
+ pkg++;
+ else
+ pkg = (char *)pkg_url; /* should not happen */
+
+ if (parsable) {
+ printf(MSG_DOWNLOAD_START);
+ } else {
+ printf(MSG_DOWNLOADING, pkg);
+ fflush(stdout);
+ start_progress_meter(pkg, st.size, &statsize);
+ }
+
+ while (written < st.size) {
+ if ((fetched = fetchIO_read(f, buf, sizeof(buf))) == 0)
+ break;
+ if (fetched == -1 && errno == EINTR)
+ continue;
+ if (fetched == -1)
+ errx(EXIT_FAILURE, "fetch failure: %s",
+ fetchLastErrString);
+
+ statsize += fetched;
+ size = fetched;
+
+ for (ptr = buf; size > 0; ptr += wrote, size -= wrote) {
+ if ((wrote = fwrite(ptr, 1, size, fp)) < size) {
+ if (ferror(fp) && errno == EINTR)
+ clearerr(fp);
+ else
+ break;
+ }
+ written += wrote;
+ }
+ }
+
+ if (parsable)
+ printf(MSG_DOWNLOAD_END);
+ else
+ stop_progress_meter();
+
+ fetchIO_close(f);
+ fetchFreeURL(url);
+
+ if (written != st.size)
+ return -1;
+
+ return written;
+}
View
@@ -228,6 +228,7 @@ extern Preflist **preflist;
/* download.c*/
Dlfile *download_file(char *, time_t *);
+ssize_t download_pkg(char *, FILE *);
/* summary.c */
int update_db(int, char **);
void split_repos(void);

0 comments on commit 0e421ba

Please sign in to comment.