Modified manifest format and database schema.

The struct pkg_file no longer has a size attribute.
The struct pkg has now a flatsize attribute, which represents the size
of the uncompressed package. This attribute is used instead of computing
the value with the old size of files. The attribute is computed in pkg
register and stored in the database, and emited in the manifest.

We also emit @file lines into the manifest so we dont have to read
the whole archive to get informations about a package.

docs/TODO

@@ -26,12 +26,15 @@ subcommands:
 	  considering the files are already installed
 
 - genrepo:
-	- takes only one argument and generate two cache files (maybe more for
+	- takes only one argument and generate two databases files (maybe more for
 	  the time we will be willing to work with UPDATING and MOVED) the first
 	  repo contain every informations concerning a package except files
 	  (which will be in the second one) + it will compute and add a sha256
-	  sum for each package. the cache will be in tcdb format compressed in
-	  xz format.
+	  sum for each package.
+	  the database will be a sqlite file compressed with the xz format.
+	  the database will be signed so we can trust the sha256 of the
+	  packages, so if a package has the expected hash, it is considered
+	  trusted.
 
 - search:
 	- will search both the remote repository to gives informations

libpkg/pkg.c

@@ -130,6 +130,15 @@ pkg_set_from_file(struct pkg *pkg, pkg_attr attr, const char *path)
 	return (ret);
 }
 
+int64_t
+pkg_flatsize(struct pkg *pkg)
+{
+	if (pkg == NULL)
+		return (-1);
+
+	return (pkg->flatsize);
+}
+
 struct pkg_script **
 pkg_scripts(struct pkg *pkg)
 {
@@ -227,7 +236,6 @@ pkg_open(const char *path, struct pkg **pkg_p, int query_flags)
 	struct archive_entry *ae;
 	struct pkg *pkg;
 	struct pkg_script *script;
-	struct pkg_file *file = NULL;
 	int ret;
 	int64_t size;
 	char *manifest;
@@ -272,12 +280,17 @@ pkg_open(const char *path, struct pkg **pkg_p, int query_flags)
 	while ((ret = archive_read_next_header(a, &ae)) == ARCHIVE_OK) {
 		fpath = archive_entry_pathname(ae);
 
+		if (fpath[0] != '+')
+			break;
+
 		if (strcmp(fpath, "+MANIFEST") == 0) {
 			size = archive_entry_size(ae);
 			manifest = calloc(1, size + 1);
 			archive_read_data(a, manifest, size);
-			pkg_parse_manifest(pkg, manifest);
+			ret = pkg_parse_manifest(pkg, manifest);
 			free(manifest);
+			if (ret != EPKG_OK)
+				goto error;
 		}
 
 #define COPY_FILE(fname, dest)												\
@@ -313,18 +326,12 @@ pkg_open(const char *path, struct pkg **pkg_p, int query_flags)
 		COPY_SCRIPT("+INSTALL", PKG_SCRIPT_INSTALL)
 		COPY_SCRIPT("+DEINSTALL", PKG_SCRIPT_DEINSTALL)
 		COPY_SCRIPT("+UPGRADE", PKG_SCRIPT_UPGRADE)
-
-		if (fpath[0] == '+')
-			continue;
-
-		pkg_file_new(&file);
-		strlcpy(file->path, archive_entry_pathname(ae), sizeof(file->path));
-		file->size = archive_entry_size(ae);
-		array_append(&pkg->files, file);
 	}
 
-	if (ret != ARCHIVE_EOF)
+	if (ret != ARCHIVE_OK && ret != ARCHIVE_EOF) {
+		warnx("libarchive: %s", archive_error_string(a));
 		goto error;
+	}
 
 	archive_read_finish(a);
 	return (EPKG_OK);
@@ -341,8 +348,6 @@ pkg_new(struct pkg **pkg)
 	if ((*pkg = calloc(1, sizeof(struct pkg))) == NULL)
 		err(EXIT_FAILURE, "calloc()");
 
-	(*pkg)->path = NULL;
-
 	return (EPKG_OK);
 }
 
@@ -365,6 +370,9 @@ pkg_reset(struct pkg *pkg)
 	sbuf_reset(pkg->www);
 	sbuf_reset(pkg->prefix);
 
+	pkg->flatsize = 0;
+	pkg->path = NULL;
+
 	array_reset(&pkg->deps, &pkg_free_void);
 	array_reset(&pkg->rdeps, &pkg_free_void);
 	array_reset(&pkg->conflicts, &pkg_conflict_free_void);
@@ -411,6 +419,19 @@ pkg_free_void(void *p)
 		pkg_free((struct pkg*) p);
 }
 
+int
+pkg_setflatsize(struct pkg *pkg, int64_t size)
+{
+	if (pkg == NULL)
+		return (EPKG_NULL_PKG);
+
+	if (size <= 0)
+		return (EPKG_FATAL);
+
+	pkg->flatsize = size;
+	return (EPKG_OK);
+}
+
 int
 pkg_addscript(struct pkg *pkg, const char *path)
 {
@@ -535,7 +556,7 @@ pkg_adddep(struct pkg *pkg, const char *name, const char *origin, const char *ve
 }
 
 int
-pkg_addfile(struct pkg *pkg, const char *path, const char *sha256, int64_t sz)
+pkg_addfile(struct pkg *pkg, const char *path, const char *sha256)
 {
 	struct pkg_file *file;
 
@@ -552,8 +573,6 @@ pkg_addfile(struct pkg *pkg, const char *path, const char *sha256, int64_t sz)
 	if (sha256 != NULL)
 		strlcpy(file->sha256, sha256, sizeof(file->sha256));
 
-	file->size = sz;
-
 	array_init(&pkg->files, 10);
 	array_append(&pkg->files, file);
 

libpkg/pkg.h

@@ -167,14 +167,19 @@ void pkg_free(struct pkg *);
  */
 int pkg_open(const char *path, struct pkg **p, int flags);
 
+pkg_t pkg_type(struct pkg *);
+
 /**
  * Generic getter for simple attributes.
  * @return NULL-terminated string.
  * @warning May return a NULL pointer.
  */
 const char *pkg_get(struct pkg *, pkg_attr);
 
-pkg_t pkg_type(struct pkg *);
+/**
+ * Returns the size of the uncompressed package.
+ */
+int64_t pkg_flatsize(struct pkg *);
 
 /**
  * @return NULL-terminated array of pkg.
@@ -248,8 +253,13 @@ int pkg_set(struct pkg *, pkg_attr, const char *);
  */
 int pkg_set_from_file(struct pkg *, pkg_attr, const char *);
 
+/**
+ * Set the uncompressed size of the package.
+ */
+int pkg_setflatsize(struct pkg *, int64_t);
+
 int pkg_adddep(struct pkg *, const char *, const char *, const char *);
-int pkg_addfile(struct pkg *, const char *, const char *, int64_t);
+int pkg_addfile(struct pkg *, const char *, const char *);
 int pkg_addconflict(struct pkg *, const char *);
 int pkg_addexec(struct pkg *, const char *, pkg_exec_t);
 int pkg_addscript(struct pkg *, const char *);
@@ -265,7 +275,6 @@ void pkg_file_reset(struct pkg_file *);
 void pkg_file_free(struct pkg_file *);
 const char * pkg_file_path(struct pkg_file *);
 const char * pkg_file_sha256(struct pkg_file *);
-int64_t pkg_file_size(struct pkg_file *);
 
 /* pkg_conflict */
 int pkg_conflict_new(struct pkg_conflict **);

libpkg/pkg_file.c

@@ -15,19 +15,12 @@ pkg_file_sha256(struct pkg_file *file)
 	return (file->sha256);
 }
 
-int64_t
-pkg_file_size(struct pkg_file *file)
-{
-	return (file->size);
-}
-
 int
 pkg_file_new(struct pkg_file **file)
 {
 	if ((*file = calloc(1, sizeof(struct pkg_file))) == NULL)
 		return (EPKG_FATAL);
 
-	(*file)->size = -1;
 	return (EPKG_OK);
 }
 
@@ -36,7 +29,6 @@ pkg_file_reset(struct pkg_file *file)
 {
 	file->path[0] = '\0';
 	file->sha256[0] = '\0';
-	file->size = -1;
 }
 
 void

libpkg/pkg_manifest.c

@@ -1,11 +1,13 @@
+#include <sys/types.h>
+#include <sys/sbuf.h>
+
 #include <assert.h>
 #include <ctype.h>
 #include <err.h>
+#include <inttypes.h>
 #include <stdbool.h>
 #include <stdlib.h>
 #include <string.h>
-#include <sys/types.h>
-#include <sys/sbuf.h>
 
 #include "pkg_util.h"
 #include "pkg.h"
@@ -18,13 +20,15 @@ static int m_parse_arch(struct pkg *pkg, char *buf);
 static int m_parse_osversion(struct pkg *pkg, char *buf);
 static int m_parse_www(struct pkg *pkg, char *buf);
 static int m_parse_comment(struct pkg *pkg, char *buf);
+static int m_parse_flatsize(struct pkg *pkg, char *buf);
 static int m_parse_option(struct pkg *pkg, char *buf);
 static int m_parse_dep(struct pkg *pkg, char *buf);
 static int m_parse_conflict(struct pkg *pkg, char *buf);
 static int m_parse_maintainer(struct pkg *pkg, char *buf);
 static int m_parse_exec(struct pkg *pkg, char *buf);
 static int m_parse_unexec(struct pkg *pkg, char *buf);
 static int m_parse_prefix(struct pkg *pkg, char *buf);
+static int m_parse_file(struct pkg *pkg, char *buf);
 static int m_parse_set_string(struct pkg *pkg, char *buf, pkg_attr attr);
 
 #define MANIFEST_FORMAT_KEY "@pkg_format_version"
@@ -40,13 +44,15 @@ static struct manifest_key {
 	{ "@osversion", m_parse_osversion},
 	{ "@www", m_parse_www},
 	{ "@comment", m_parse_comment},
+	{ "@flatsize", m_parse_flatsize},
 	{ "@option", m_parse_option},
 	{ "@dep", m_parse_dep},
 	{ "@conflict", m_parse_conflict},
 	{ "@maintainer", m_parse_maintainer},
 	{ "@exec", m_parse_exec},
 	{ "@unexec", m_parse_unexec},
 	{ "@prefix", m_parse_prefix},
+	{ "@file", m_parse_file},
 };
 
 #define manifest_key_len (int)(sizeof(manifest_key)/sizeof(manifest_key[0]))
@@ -115,6 +121,20 @@ m_parse_comment(struct pkg *pkg, char *buf)
 	return (m_parse_set_string(pkg, buf, PKG_COMMENT));
 }
 
+static int
+m_parse_flatsize(struct pkg *pkg, char *buf)
+{
+	int64_t size;
+
+	size = strtoimax(buf, NULL, 10);
+
+	if (size <= 0)
+		return (EPKG_FATAL);
+
+	pkg_setflatsize(pkg, size);
+	return (EPKG_OK);
+}
+
 static int
 m_parse_exec(struct pkg *pkg, char *buf)
 {
@@ -212,6 +232,28 @@ m_parse_conflict(struct pkg *pkg, char *buf)
 	return (EPKG_OK);
 }
 
+static int
+m_parse_file(struct pkg *pkg, char *buf)
+{
+	const char *path;
+	const char *sha256;
+
+	while (isspace(*buf))
+		buf++;
+
+	if (split_chr(buf, ' ') != 1)
+		return (EPKG_FATAL);
+
+	path = buf;
+
+	buf += strlen(path) + 1;
+	sha256 = buf;
+
+	pkg_addfile(pkg, path, sha256);
+
+	return (EPKG_OK);
+}
+
 int
 pkg_parse_manifest(struct pkg *pkg, char *buf)
 {
@@ -235,8 +277,13 @@ pkg_parse_manifest(struct pkg *pkg, char *buf)
 	for (i = 1; i <= nbel; i++) {
 		for (j = 0; j < manifest_key_len; j++) {
 			if (STARTS_WITH(buf_ptr, manifest_key[j].key)) {
-				if (manifest_key[j].parse(pkg, buf_ptr + strlen(manifest_key[j].key)) != EPKG_OK)
+				if (manifest_key[j].parse(pkg, buf_ptr +
+					strlen(manifest_key[j].key)) != EPKG_OK) {
+
+					warnx("Error while parsing %s at line %d",
+						  manifest_key[j].key, i + 1);
 					return (EPKG_FATAL);
+				}
 				break;
 			}
 		}
@@ -258,6 +305,7 @@ pkg_emit_manifest(struct pkg *pkg, char **dest)
 	struct pkg_conflict **conflicts;
 	struct pkg_exec **execs;
 	struct pkg_option **options;
+	struct pkg_file **files;
 	int i;
 	int len = 0;
 
@@ -272,7 +320,8 @@ pkg_emit_manifest(struct pkg *pkg, char **dest)
 			"@osversion %s\n"
 			"@www %s\n"
 			"@maintainer %s\n"
-			"@prefix %s\n",
+			"@prefix %s\n"
+			"@flatsize %" PRId64 "\n",
 			pkg_get(pkg, PKG_NAME),
 			pkg_get(pkg, PKG_VERSION),
 			pkg_get(pkg, PKG_ORIGIN),
@@ -281,7 +330,8 @@ pkg_emit_manifest(struct pkg *pkg, char **dest)
 			pkg_get(pkg, PKG_OSVERSION),
 			pkg_get(pkg, PKG_WWW),
 			pkg_get(pkg, PKG_MAINTAINER) ? pkg_get(pkg, PKG_MAINTAINER) : "UNKNOWN",
-			pkg_get(pkg, PKG_PREFIX)
+			pkg_get(pkg, PKG_PREFIX),
+			pkg_flatsize(pkg)
 			);
 
 	if ((deps = pkg_deps(pkg)) != NULL) {
@@ -316,6 +366,13 @@ pkg_emit_manifest(struct pkg *pkg, char **dest)
 		}
 	}
 
+	if ((files = pkg_files(pkg)) != NULL) {
+		for (i = 0; files[i] != NULL; i++) {
+			sbuf_printf(manifest, "@file %s %s\n", pkg_file_path(files[i]),
+						pkg_file_sha256(files[i]));
+		}
+	}
+
 	sbuf_finish(manifest);
 	len = sbuf_len(manifest);
 	*dest = strdup(sbuf_data(manifest));

libpkg/pkg_ports.c

@@ -87,7 +87,7 @@ ports_parse_plist(struct pkg *pkg, char *plist)
 				p = NULL;
 			}
 
-			ret += pkg_addfile(pkg, path, p, -1);
+			ret += pkg_addfile(pkg, path, p);
 		}
 
 		if (i != nbel) {

libpkg/pkg_private.h

@@ -21,6 +21,7 @@ struct pkg {
 	struct sbuf *www;
 	struct sbuf *err;
 	struct sbuf *prefix;
+	int64_t flatsize;
 	struct array deps;
 	struct array rdeps;
 	struct array conflicts;
@@ -49,7 +50,6 @@ struct pkg_exec {
 struct pkg_file {
 	char path[MAXPATHLEN];
 	char sha256[65];
-	int64_t size;
 };
 
 struct pkg_option {