Permalink
Browse files

Stream pkg INSERTs rather than caching everything.

Use libarchive (which we were linking against but not using) to handle
streaming the remote pkg_summary.  The copy of decompress.c is no longer
required.

We now stream one pkg_info record at a time, rather than loading the
entire contents in and then processing them all in one go.  They are
still processed as part of the same commit, so there is no performance
penalty (if anything this version is slightly faster).

Using the SmartOS 2015Q1 x86_64 repository as a test corpus, heap usage
for a clean "pkgin update" with this change goes down from 75MB to 29MB.
  • Loading branch information...
jperkin committed Jul 16, 2015
1 parent 8530c8a commit bb51acbab56929b7c40bb3a450e276de1a268a3d
Showing with 266 additions and 447 deletions.
  1. +1 −2 Makefile.in
  2. +0 −196 external/decompress.c
  3. +0 −3 external/lib.h
  4. +265 −246 summary.c
View
@@ -8,8 +8,7 @@ SRCS= main.c summary.c tools.c pkgindb.c depends.c actions.c \
pkgindb_queries.c pkg_str.c sqlite_callbacks.c selection.c \
pkg_check.c pkg_infos.c preferred.c
# included from libinstall
-SRCS+= automatic.c decompress.c dewey.c fexec.c global.c \
- opattern.c pkgdb.c var.c
+SRCS+= automatic.c dewey.c fexec.c global.c opattern.c pkgdb.c var.c
# included from openssh
SRCS+= progressmeter.c
View
@@ -1,196 +0,0 @@
-/*-
- * Copyright (c) 2008 Joerg Sonnenberger <joerg@NetBSD.org>.
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- *
- * 1. Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- * notice, this list of conditions and the following disclaimer in
- * the documentation and/or other materials provided with the
- * distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
- * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
- * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
- * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
- * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
- * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
- * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
- * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
- * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- */
-
-#if HAVE_CONFIG_H
-#include "config.h"
-#endif
-
-#if HAVE_NBCOMPAT_H
-#include <nbcompat.h>
-#endif
-
-#if HAVE_SYS_CDEFS_H
-#include <sys/cdefs.h>
-#endif
-
-__RCSID("$NetBSD: decompress.c,v 1.1.1.1 2008/09/30 19:00:26 joerg Exp $");
-
-#ifdef BOOTSTRAP
-#include "lib.h"
-
-int
-decompress_buffer(const char *input, size_t input_len, char **output,
- size_t *output_len)
-{
- return 0;
-}
-
-#else
-
-#include <bzlib.h>
-#if HAVE_ERR_H
-#include <err.h>
-#endif
-#include <limits.h>
-#include <stdlib.h>
-#include <zlib.h>
-
-#include "lib.h"
-
-static void
-decompress_bzip2(const char *in, size_t in_len, char **out, size_t *out_len)
-{
- bz_stream stream;
- size_t output_produced;
-
- if (in_len < SSIZE_MAX / 10)
- *out_len = in_len * 10;
- else
- *out_len = in_len;
- if ((*out = malloc(*out_len + 1)) == NULL)
- err(EXIT_FAILURE, "malloc failed");
-
- stream.next_in = __UNCONST(in);
- stream.avail_in = in_len;
- stream.next_out = *out;
- stream.avail_out = *out_len;
- output_produced = 0;
- stream.bzalloc = NULL;
- stream.bzfree = NULL;
- stream.opaque = NULL;
-
- if (BZ2_bzDecompressInit(&stream, 0, 0) != BZ_OK)
- errx(EXIT_FAILURE, "BZ2_bzDecompressInit failed");
-
- for (;;) {
- switch (BZ2_bzDecompress(&stream)) {
- case BZ_STREAM_END:
- if (BZ2_bzDecompressEnd(&stream) != Z_OK)
- errx(EXIT_FAILURE, "inflateEnd failed");
- output_produced = *out_len - stream.avail_out;
- *out = realloc(*out, output_produced + 1);
- if (*out == NULL)
- err(EXIT_FAILURE, "realloc failed");
- *out_len = output_produced;
- (*out)[*out_len] = '\0';
- return;
- case BZ_OK:
- output_produced = *out_len - stream.avail_out;
-#ifdef _MINIX /* buggy SSIZE_MAX on MINIX */
- *out_len *= 2;
-#else
- if (*out_len <= SSIZE_MAX / 2)
- *out_len *= 2;
- else
- errx(EXIT_FAILURE, "input too large");
-#endif
- *out = realloc(*out, *out_len + 1);
- stream.next_out = *out + output_produced;
- stream.avail_out = *out_len - output_produced;
- break;
- default:
- errx(EXIT_FAILURE, "inflate failed");
- }
- }
-}
-
-static void
-decompress_zlib(const char *in, size_t in_len, char **out, size_t *out_len)
-{
- z_stream stream;
- size_t output_produced;
-
- if (in_len < SSIZE_MAX / 10)
- *out_len = in_len * 10;
- else
- *out_len = in_len;
- if ((*out = malloc(*out_len + 1)) == NULL)
- err(EXIT_FAILURE, "malloc failed");
-
- stream.next_in = __UNCONST(in);
- stream.avail_in = in_len;
- stream.next_out = (unsigned char *)*out;
- stream.avail_out = *out_len;
- output_produced = 0;
- stream.zalloc = Z_NULL;
- stream.zfree = Z_NULL;
- stream.opaque = NULL;
-
- if (inflateInit2(&stream, 47) != Z_OK)
- errx(EXIT_FAILURE, "inflateInit failed");
-
- for (;;) {
- switch (inflate(&stream, Z_FINISH)) {
- case Z_STREAM_END:
- if (inflateEnd(&stream) != Z_OK)
- errx(EXIT_FAILURE, "inflateEnd failed");
- output_produced = *out_len - stream.avail_out;
- *out = realloc(*out, output_produced + 1);
- if (*out == NULL)
- err(EXIT_FAILURE, "realloc failed");
- *out_len = output_produced;
- (*out)[*out_len] = '\0';
- return;
- case Z_OK:
- output_produced = *out_len - stream.avail_out;
- if (*out_len < SSIZE_MAX / 2)
- *out_len *= 2;
- else if (*out_len == SSIZE_MAX - 1)
- errx(EXIT_FAILURE, "input too large");
- else
- *out_len = SSIZE_MAX - 1;
- *out = realloc(*out, *out_len + 1);
- stream.next_out = (unsigned char *)*out + output_produced;
- stream.avail_out = *out_len - output_produced;
- break;
- default:
- errx(EXIT_FAILURE, "inflate failed");
- }
- }
-}
-
-int
-decompress_buffer(const char *input, size_t input_len, char **output,
- size_t *output_len)
-{
- if (input_len < 4)
- return 0;
- if (input[0] == 'B' && input[1] == 'Z' && input[2] == 'h' &&
- input[3] >= '1' && input[3] <= '9') {
- /* Normal bzip2. */
- decompress_bzip2(input, input_len, output, output_len);
- } else if (input[0] == 037 && (unsigned char)input[1] == 139 &&
- input[2] == 8 && (input[3] & 0xe0) == 0) {
- /* gzip header with Deflate method */
- decompress_zlib(input, input_len, output, output_len);
- } else /* plain text */
- return 0;
- return 1;
-}
-#endif /* BOOTSTRAP */
View
@@ -441,9 +441,6 @@ lpkg_t *alloc_lpkg(const char *);
lpkg_t *find_on_queue(lpkg_head_t *, const char *);
void free_lpkg(lpkg_t *);
-/* Extract input if compressed to NUL terminated buffer (not counted) */
-int decompress_buffer(const char *, size_t, char **, size_t *);
-
/* Parse NUL terminated inputed, argument is strlen of the input */
struct pkg_vulnerabilities *parse_pkg_vulnerabilities(const char *, size_t, int);
/* Read pkg_vulnerabilities from file */
Oops, something went wrong.

0 comments on commit bb51acb

Please sign in to comment.