Stream package fetches directly to the local file.

Previously package downloads would go through download_file() and cache
the entire file in RAM before writing to disk.  Introduce a new
download_pkg() function which streams 4K at a time and significantly
reduces memory usage.

actions.c

@@ -63,8 +63,8 @@ pkg_download(Plisthead *installhead)
 	FILE		*fp;
 	Pkglist  	*pinstall;
 	struct stat	st;
-	Dlfile		*dlpkg;
 	char		pkg_fs[BUFSIZ], pkg_url[BUFSIZ], query[BUFSIZ];
+	ssize_t		size;
 	int		rc = EXIT_SUCCESS;
 
 	printf(MSG_DOWNLOAD_PKGS);
@@ -109,7 +109,7 @@ pkg_download(Plisthead *installhead)
 		if ((fp = fopen(pkg_fs, "w")) == NULL)
 			err(EXIT_FAILURE, MSG_ERR_OPEN, pkg_fs);
 
-		if ((dlpkg = download_file(pkg_url, NULL)) == NULL) {
+		if ((size = download_pkg(pkg_url, fp)) == -1) {
 			fprintf(stderr, MSG_PKG_NOT_AVAIL, pinstall->depend);
 			rc = EXIT_FAILURE;
 
@@ -122,12 +122,11 @@ pkg_download(Plisthead *installhead)
 			continue;
 		}
 
-		fwrite(dlpkg->buf, dlpkg->size, 1, fp);
 		fclose(fp);
-
-		XFREE(dlpkg->buf);
-		XFREE(dlpkg);
-
+		if (size != pinstall->file_size) {
+			(void)unlink(pkg_fs);
+			errx(EXIT_FAILURE, "download mismatch: %s", pkg_fs);
+		}
 	} /* download loop */
 
 	return rc;

download.c

@@ -134,3 +134,78 @@ download_file(char *str_url, time_t *db_mtime)
 
 	return file;
 }
+
+/*
+ * Download a package to the local cache.
+ */
+ssize_t
+download_pkg(char *pkg_url, FILE *fp)
+{
+	struct url_stat st;
+	size_t size, wrote;
+	ssize_t fetched, written = 0;
+	off_t statsize = 0;
+	struct url *url;
+	fetchIO *f = NULL;
+	char buf[4096];
+	char *pkg, *ptr;
+
+	if ((url = fetchParseURL(pkg_url)) == NULL)
+		errx(EXIT_FAILURE, "%s: parse failure", pkg_url);
+
+	if ((f = fetchXGet(url, &st, "")) == NULL)
+		errx(EXIT_FAILURE, "%s: %s", pkg_url, fetchLastErrString);
+
+	/* Package not available */
+	if (st.size == -1)
+		return st.size;
+
+	if ((pkg = strrchr(pkg_url, '/')) != NULL)
+		pkg++;
+	else
+		pkg = (char *)pkg_url; /* should not happen */
+
+	if (parsable) {
+		printf(MSG_DOWNLOAD_START);
+	} else {
+		printf(MSG_DOWNLOADING, pkg);
+		fflush(stdout);
+		start_progress_meter(pkg, st.size, &statsize);
+	}
+
+	while (written < st.size) {
+		if ((fetched = fetchIO_read(f, buf, sizeof(buf))) == 0)
+			break;
+		if (fetched == -1 && errno == EINTR)
+			continue;
+		if (fetched == -1)
+			errx(EXIT_FAILURE, "fetch failure: %s",
+			    fetchLastErrString);
+
+		statsize += fetched;
+		size = fetched;
+
+		for (ptr = buf; size > 0; ptr += wrote, size -= wrote) {
+			if ((wrote = fwrite(ptr, 1, size, fp)) < size) {
+				if (ferror(fp) && errno == EINTR)
+					clearerr(fp);
+				else
+					break;
+			}
+			written += wrote;
+		}
+	}
+
+	if (parsable)
+		printf(MSG_DOWNLOAD_END);
+	else
+		stop_progress_meter();
+
+	fetchIO_close(f);
+	fetchFreeURL(url);
+
+	if (written != st.size)
+		return -1;
+
+	return written;
+}

pkgin.h

@@ -228,6 +228,7 @@ extern Preflist		**preflist;
 
 /* download.c*/
 Dlfile		*download_file(char *, time_t *);
+ssize_t		download_pkg(char *, FILE *);
 /* summary.c */
 int		update_db(int, char **);
 void		split_repos(void);