From b9bd5f7476cc713ff0bb70c86c1f1e89acc5a133 Mon Sep 17 00:00:00 2001 From: Jonathan Dieter Date: Sun, 6 Jan 2019 19:21:04 +0000 Subject: [PATCH 1/6] zck_end_chunk returns number of bytes written or -1 for error Signed-off-by: Jonathan Dieter --- src/compression_wrapper.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/compression_wrapper.c b/src/compression_wrapper.c index 43f213c8..e5b58d24 100644 --- a/src/compression_wrapper.c +++ b/src/compression_wrapper.c @@ -880,7 +880,7 @@ cr_close(CR_FILE *cr_file, GError **err) zckCtx *zck = (zckCtx *) cr_file->FILE; ret = CRE_OK; if (cr_file->mode == CR_CW_MODE_WRITE) { - if(!zck_end_chunk(zck)) { + if(zck_end_chunk(zck) < 0) { ret = CRE_ZCK; g_set_error(err, ERR_DOMAIN, CRE_ZCK, "Unable to end final chunk: %s", zck_get_error(zck)); From defe0620cf9841dc099f3d23a99f55e7c6baaab7 Mon Sep 17 00:00:00 2001 From: Jonathan Dieter Date: Sun, 13 Jan 2019 22:22:20 +0000 Subject: [PATCH 2/6] Add support for zchunking extra metadata files Signed-off-by: Jonathan Dieter --- src/compression_wrapper.c | 52 ++++++++++++- src/compression_wrapper.h | 8 ++ src/createrepo_c.c | 95 ++++++++++++++++++++++- src/deltarpms.c | 14 ++++ src/deltarpms.h.in | 1 + src/mergerepo_c.c | 8 +- src/misc.c | 42 ++++++++++ src/misc.h | 13 +++- src/modifyrepo_c.c | 21 +++++ src/modifyrepo_shared.c | 146 ++++++++++++++++++++++++----------- src/modifyrepo_shared.h | 3 + src/python/misc-py.c | 2 +- src/python/repomdrecord-py.c | 7 +- src/repomd.c | 36 +++++++++ src/repomd.h | 2 + src/sqliterepo_c.c | 6 +- src/threads.c | 9 +++ src/threads.h | 6 ++ 18 files changed, 407 insertions(+), 64 deletions(-) diff --git a/src/compression_wrapper.c b/src/compression_wrapper.c index e5b58d24..8dbb25b6 100644 --- a/src/compression_wrapper.c +++ b/src/compression_wrapper.c @@ -1405,7 +1405,8 @@ cr_end_chunk(CR_FILE *cr_file, GError **err) ssize_t wb = zck_end_chunk(zck); if(wb < 0) { g_set_error(err, ERR_DOMAIN, CRE_ZCK, - "Error ending chunk"); + "Error ending chunk: %s", + zck_get_error(zck)); return CR_CW_ERR; } ret = wb; @@ -1430,6 +1431,55 @@ cr_end_chunk(CR_FILE *cr_file, GError **err) return ret; } +int +cr_set_autochunk(CR_FILE *cr_file, gboolean auto_chunk, GError **err) +{ + int ret = CRE_OK; + + assert(cr_file); + assert(!err || *err == NULL); + + if (cr_file->mode != CR_CW_MODE_WRITE) { + g_set_error(err, ERR_DOMAIN, CRE_BADARG, + "File is not opened in write mode"); + return CR_CW_ERR; + } + + switch (cr_file->type) { + case (CR_CW_NO_COMPRESSION): // --------------------------------------- + case (CR_CW_GZ_COMPRESSION): // --------------------------------------- + case (CR_CW_BZ2_COMPRESSION): // -------------------------------------- + case (CR_CW_XZ_COMPRESSION): // --------------------------------------- + break; + case (CR_CW_ZCK_COMPRESSION): { // ------------------------------------ +#ifdef WITH_ZCHUNK + zckCtx *zck = (zckCtx *) cr_file->FILE; + if(!zck_set_ioption(zck, ZCK_MANUAL_CHUNK, !auto_chunk)) { + g_set_error(err, ERR_DOMAIN, CRE_ZCK, + "Error setting auto_chunk: %s", + zck_get_error(zck)); + return CR_CW_ERR; + } + break; +#else + g_set_error(err, ERR_DOMAIN, CRE_IO, "createrepo_c wasn't compiled " + "with zchunk support"); + break; +#endif // WITH_ZCHUNK + } + + default: // ----------------------------------------------------------- + g_set_error(err, ERR_DOMAIN, CRE_BADARG, + "Bad compressed file type"); + return CR_CW_ERR; + break; + } + + assert(!err || (ret == CR_CW_ERR && *err != NULL) + || (ret != CR_CW_ERR && *err == NULL)); + + return ret; +} int cr_printf(GError **err, CR_FILE *cr_file, const char *format, ...) diff --git a/src/compression_wrapper.h b/src/compression_wrapper.h index 52782ea2..d840ec6f 100644 --- a/src/compression_wrapper.h +++ b/src/compression_wrapper.h @@ -182,6 +182,14 @@ int cr_puts(CR_FILE *cr_file, const char *str, GError **err); */ int cr_end_chunk(CR_FILE *cr_file, GError **err); +/** Set zchunk auto-chunk algorithm. Must be done before first byte is written + * @param cr_file CR_FILE pointer + * @param auto_chunk Whether auto-chunking should be enabled + * @param err GError ** + * @return CRE_OK or CR_CW_ERR + */ +int cr_set_autochunk(CR_FILE *cr_file, gboolean auto_chunk, GError **err); + /** Writes a formatted string into the cr_file. * @param err GError ** * @param cr_file CR_FILE pointer diff --git a/src/createrepo_c.c b/src/createrepo_c.c index 0b38827a..5474aad7 100644 --- a/src/createrepo_c.c +++ b/src/createrepo_c.c @@ -1073,8 +1073,11 @@ main(int argc, char **argv) cr_RepomdRecord *oth_zck_rec = NULL; cr_RepomdRecord *groupfile_rec = NULL; cr_RepomdRecord *compressed_groupfile_rec = NULL; + cr_RepomdRecord *compressed_groupfile_zck_rec = NULL; cr_RepomdRecord *updateinfo_rec = NULL; + cr_RepomdRecord *updateinfo_zck_rec = NULL; cr_RepomdRecord *prestodelta_rec = NULL; + cr_RepomdRecord *prestodelta_zck_rec = NULL; // XML cr_repomd_record_load_contentstat(pri_xml_rec, pri_stat); @@ -1115,6 +1118,7 @@ main(int argc, char **argv) compressed_groupfile_rec, cmd_options->repomd_checksum_type, groupfile_compression, + NULL, &tmp_err); if (tmp_err) { g_critical("Cannot process groupfile %s: %s", @@ -1177,21 +1181,21 @@ main(int argc, char **argv) pri_db_name, sqlite_compression, cmd_options->repomd_checksum_type, - 1, NULL); + NULL, FALSE, 1, NULL); g_thread_pool_push(compress_pool, pri_db_task, NULL); fil_db_task = cr_compressiontask_new(fil_db_filename, fil_db_name, sqlite_compression, cmd_options->repomd_checksum_type, - 1, NULL); + NULL, FALSE, 1, NULL); g_thread_pool_push(compress_pool, fil_db_task, NULL); oth_db_task = cr_compressiontask_new(oth_db_filename, oth_db_name, sqlite_compression, cmd_options->repomd_checksum_type, - 1, NULL); + NULL, FALSE, 1, NULL); g_thread_pool_push(compress_pool, oth_db_task, NULL); g_thread_pool_free(compress_pool, FALSE, TRUE); @@ -1286,6 +1290,50 @@ main(int argc, char **argv) cr_repomdrecordfilltask_free(pri_zck_fill_task, NULL); cr_repomdrecordfilltask_free(fil_zck_fill_task, NULL); cr_repomdrecordfilltask_free(oth_zck_fill_task, NULL); + + // Group file + if (groupfile && groupfile_compression != CR_CW_ZCK_COMPRESSION) { + compressed_groupfile_zck_rec = cr_repomd_record_new("group_gz_zck", groupfile); + cr_repomd_record_compress_and_fill(groupfile_rec, + compressed_groupfile_zck_rec, + cmd_options->repomd_checksum_type, + CR_CW_ZCK_COMPRESSION, + cmd_options->zck_dict_dir, + &tmp_err); + if (tmp_err) { + g_critical("Cannot process groupfile %s: %s", + groupfile, tmp_err->message); + g_clear_error(&tmp_err); + exit(EXIT_FAILURE); + } + } + + // Updateinfo + if (updateinfo) { + cr_CompressionType com_type = cr_detect_compression(updateinfo, &tmp_err); + if (tmp_err) { + g_critical("Cannot detect compression type of %s: %s", + updateinfo, tmp_err->message); + g_clear_error(&tmp_err); + exit(EXIT_FAILURE); + } + /* Only create updateinfo_zck if updateinfo isn't already zchunk */ + if(com_type != CR_CW_ZCK_COMPRESSION) { + updateinfo_zck_rec = cr_repomd_record_new("updateinfo_zck", updateinfo); + cr_repomd_record_compress_and_fill(updateinfo_rec, + updateinfo_zck_rec, + cmd_options->repomd_checksum_type, + CR_CW_ZCK_COMPRESSION, + cmd_options->zck_dict_dir, + &tmp_err); + if (tmp_err) { + g_critical("Cannot process updateinfo %s: %s", + updateinfo, tmp_err->message); + g_clear_error(&tmp_err); + exit(EXIT_FAILURE); + } + } + } } cr_contentstat_free(pri_zck_stat, NULL); cr_contentstat_free(fil_zck_stat, NULL); @@ -1296,9 +1344,13 @@ main(int argc, char **argv) if (cmd_options->deltas) { gchar *filename, *outdeltadir = NULL; gchar *prestodelta_xml_filename = NULL; + gchar *prestodelta_zck_filename = NULL; + GHashTable *ht_oldpackagedirs = NULL; cr_XmlFile *prestodelta_cr_file = NULL; + cr_XmlFile *prestodelta_cr_zck_file = NULL; cr_ContentStat *prestodelta_stat = NULL; + cr_ContentStat *prestodelta_zck_stat = NULL; filename = g_strconcat("prestodelta.xml", prestodelta_compression_suffix, @@ -1357,10 +1409,31 @@ main(int argc, char **argv) g_clear_error(&tmp_err); goto deltaerror; } + if(cmd_options->zck_compression && prestodelta_compression != CR_CW_ZCK_COMPRESSION) { + filename = g_strconcat("prestodelta.xml", + cr_compression_suffix(CR_CW_ZCK_COMPRESSION), + NULL); + prestodelta_zck_filename = g_build_filename(tmp_out_repo, + filename, + NULL); + g_free(filename); + + prestodelta_zck_stat = cr_contentstat_new(cmd_options->repomd_checksum_type, NULL); + prestodelta_cr_zck_file = cr_xmlfile_sopen_prestodelta(prestodelta_zck_filename, + CR_CW_ZCK_COMPRESSION, + prestodelta_zck_stat, + &tmp_err); + if (!prestodelta_cr_zck_file) { + g_critical("Cannot open %s: %s", prestodelta_zck_filename, tmp_err->message); + g_clear_error(&tmp_err); + goto deltaerror; + } + } ret = cr_deltarpms_generate_prestodelta_file( outdeltadir, prestodelta_cr_file, + prestodelta_cr_zck_file, //cmd_options->checksum_type, CR_CHECKSUM_SHA256, // Createrepo always uses SHA256 cmd_options->workers, @@ -1375,19 +1448,29 @@ main(int argc, char **argv) cr_xmlfile_close(prestodelta_cr_file, NULL); prestodelta_cr_file = NULL; + cr_xmlfile_close(prestodelta_cr_zck_file, NULL); + prestodelta_cr_zck_file = NULL; // 4) Prepare repomd record prestodelta_rec = cr_repomd_record_new("prestodelta", prestodelta_xml_filename); cr_repomd_record_load_contentstat(prestodelta_rec, prestodelta_stat); cr_repomd_record_fill(prestodelta_rec, cmd_options->repomd_checksum_type, NULL); + if(prestodelta_zck_stat) { + prestodelta_zck_rec = cr_repomd_record_new("prestodelta_zck", prestodelta_zck_filename); + cr_repomd_record_load_contentstat(prestodelta_zck_rec, prestodelta_zck_stat); + cr_repomd_record_fill(prestodelta_zck_rec, cmd_options->repomd_checksum_type, NULL); + } deltaerror: // 5) Cleanup g_hash_table_destroy(ht_oldpackagedirs); g_free(outdeltadir); g_free(prestodelta_xml_filename); + g_free(prestodelta_zck_filename); cr_xmlfile_close(prestodelta_cr_file, NULL); + cr_xmlfile_close(prestodelta_cr_zck_file, NULL); cr_contentstat_free(prestodelta_stat, NULL); + cr_contentstat_free(prestodelta_zck_stat, NULL); cr_slist_free_full(user_data.deltatargetpackages, (GDestroyNotify) cr_deltatargetpackage_free); } @@ -1406,8 +1489,11 @@ main(int argc, char **argv) cr_repomd_record_rename_file(oth_zck_rec, NULL); cr_repomd_record_rename_file(groupfile_rec, NULL); cr_repomd_record_rename_file(compressed_groupfile_rec, NULL); + cr_repomd_record_rename_file(compressed_groupfile_zck_rec, NULL); cr_repomd_record_rename_file(updateinfo_rec, NULL); + cr_repomd_record_rename_file(updateinfo_zck_rec, NULL); cr_repomd_record_rename_file(prestodelta_rec, NULL); + cr_repomd_record_rename_file(prestodelta_zck_rec, NULL); } // Gen xml @@ -1422,8 +1508,11 @@ main(int argc, char **argv) cr_repomd_set_record(repomd_obj, oth_zck_rec); cr_repomd_set_record(repomd_obj, groupfile_rec); cr_repomd_set_record(repomd_obj, compressed_groupfile_rec); + cr_repomd_set_record(repomd_obj, compressed_groupfile_zck_rec); cr_repomd_set_record(repomd_obj, updateinfo_rec); + cr_repomd_set_record(repomd_obj, updateinfo_zck_rec); cr_repomd_set_record(repomd_obj, prestodelta_rec); + cr_repomd_set_record(repomd_obj, prestodelta_zck_rec); int i = 0; while (cmd_options->repo_tags && cmd_options->repo_tags[i]) diff --git a/src/deltarpms.c b/src/deltarpms.c index 845b1ea8..53a9ea6f 100644 --- a/src/deltarpms.c +++ b/src/deltarpms.c @@ -846,6 +846,7 @@ gen_newpackage_xml_chunk(const char *strnevra, gboolean cr_deltarpms_generate_prestodelta_file(const gchar *drpmsdir, cr_XmlFile *f, + cr_XmlFile *zck_f, cr_ChecksumType checksum_type, gint workers, const gchar *prefix_to_strip, @@ -917,6 +918,19 @@ cr_deltarpms_generate_prestodelta_file(const gchar *drpmsdir, chunk = gen_newpackage_xml_chunk(nevra, (GSList *) value); cr_xmlfile_add_chunk(f, chunk, NULL); + + /* Write out zchunk file */ + if(zck_f) { + cr_xmlfile_add_chunk(zck_f, chunk, NULL); + cr_end_chunk(zck_f->f, &tmp_err); + if (tmp_err) { + g_free(chunk); + g_propagate_prefixed_error(err, tmp_err, + "Cannot create pool for prestodelta file generation: "); + ret = FALSE; + goto exit; + } + } g_free(chunk); } diff --git a/src/deltarpms.h.in b/src/deltarpms.h.in index 04341e15..1ee97edd 100644 --- a/src/deltarpms.h.in +++ b/src/deltarpms.h.in @@ -112,6 +112,7 @@ cr_deltarpms_scan_targetdir(const char *path, gboolean cr_deltarpms_generate_prestodelta_file(const gchar *drpmdir, cr_XmlFile *f, + cr_XmlFile *zck_f, cr_ChecksumType checksum_type, gint workers, const gchar *prefix_to_strip, diff --git a/src/mergerepo_c.c b/src/mergerepo_c.c index df49cf02..fbdde5a5 100644 --- a/src/mergerepo_c.c +++ b/src/mergerepo_c.c @@ -1554,7 +1554,7 @@ dump_merged_metadata(GHashTable *merged_hashtable, compressed_groupfile_rec, CR_CHECKSUM_SHA256, cmd_options->groupfile_compression_type, - NULL); + NULL, NULL); } @@ -1621,21 +1621,21 @@ dump_merged_metadata(GHashTable *merged_hashtable, pri_db_c_filename, cmd_options->db_compression_type, CR_CHECKSUM_SHA256, - 1, NULL); + NULL, FALSE, 1, NULL); g_thread_pool_push(compress_pool, pri_db_task, NULL); fil_db_task = cr_compressiontask_new(fil_db_filename, fil_db_c_filename, cmd_options->db_compression_type, CR_CHECKSUM_SHA256, - 1, NULL); + NULL, FALSE, 1, NULL); g_thread_pool_push(compress_pool, fil_db_task, NULL); oth_db_task = cr_compressiontask_new(oth_db_filename, oth_db_c_filename, cmd_options->db_compression_type, CR_CHECKSUM_SHA256, - 1, NULL); + NULL, FALSE, 1, NULL); g_thread_pool_push(compress_pool, oth_db_task, NULL); g_thread_pool_free(compress_pool, FALSE, TRUE); diff --git a/src/misc.c b/src/misc.c index ec8667b8..e9a5568b 100644 --- a/src/misc.c +++ b/src/misc.c @@ -432,6 +432,8 @@ cr_compress_file_with_stat(const char *src, const char *in_dst, cr_CompressionType compression, cr_ContentStat *stat, + const char *zck_dict_dir, + gboolean zck_auto_chunk, GError **err) { int ret = CRE_OK; @@ -476,6 +478,34 @@ cr_compress_file_with_stat(const char *src, goto compress_file_cleanup; } + _cleanup_free_ gchar *dict = NULL; + size_t dict_size = 0; + if(compression == CR_CW_ZCK_COMPRESSION && zck_dict_dir) { + /* Find zdict */ + _cleanup_free_ gchar *file_basename = NULL; + if(in_dst) { + _cleanup_free_ gchar *dict_base = NULL; + if(g_str_has_suffix(in_dst, ".zck")) + dict_base = g_strndup(in_dst, strlen(in_dst)-4); + else + dict_base = g_strdup(in_dst); + file_basename = g_path_get_basename(dict_base); + } else { + file_basename = g_path_get_basename(src); + } + _cleanup_free_ gchar *dict_file = cr_get_dict_file(zck_dict_dir, file_basename); + + /* Read dictionary from file */ + if(dict_file && !g_file_get_contents(dict_file, &dict, + &dict_size, &tmp_err)) { + g_set_error(err, ERR_DOMAIN, CRE_IO, + "Error reading zchunk dict %s: %s", + dict_file, tmp_err->message); + ret = CRE_IO; + goto compress_file_cleanup; + } + } + new = cr_sopen(dst, CR_CW_MODE_WRITE, compression, stat, &tmp_err); if (tmp_err) { g_debug("%s: Cannot open destination file %s", __func__, dst); @@ -483,6 +513,18 @@ cr_compress_file_with_stat(const char *src, ret = CRE_IO; goto compress_file_cleanup; } + if(compression == CR_CW_ZCK_COMPRESSION) { + if(dict && cr_set_dict(new, dict, dict_size, &tmp_err) != CRE_OK) { + ret = tmp_err->code; + g_propagate_prefixed_error(err, tmp_err, "Unable to set zdict for %s: ", dst); + return ret; + } + if(zck_auto_chunk && cr_set_autochunk(new, TRUE, &tmp_err) != CRE_OK) { + ret = tmp_err->code; + g_propagate_prefixed_error(err, tmp_err, "Unable to set auto-chunking for %s: ", dst); + return ret; + } + } while ((readed = fread(buf, 1, BUFFER_SIZE, orig)) > 0) { if (readed != BUFFER_SIZE && ferror(orig)) { diff --git a/src/misc.h b/src/misc.h index 517f6f5a..e6dd0278 100644 --- a/src/misc.h +++ b/src/misc.h @@ -168,11 +168,14 @@ gboolean cr_copy_file(const char *src, * compression suffix is used. * If dst is NULL, src + compression suffix is used) * @param COMTYPE type of compression + * @param ZCK_DICT_DIR Location of zchunk zdicts (if zchunk is enabled) + * @param ZCK_AUTO_CHUNK Whether zchunk file should be auto-chunked * @param ERR GError ** * @return cr_Error return code */ -#define cr_compress_file(SRC, DST, COMTYPE, ERR) \ - cr_compress_file_with_stat(SRC, DST, COMTYPE, NULL, ERR) +#define cr_compress_file(SRC, DST, COMTYPE, ZCK_DICT_DIR, ZCK_AUTO_CHUNK, ERR) \ + cr_compress_file_with_stat(SRC, DST, COMTYPE, NULL, ZCK_DICT_DIR, \ + ZCK_AUTO_CHUNK, ERR) /** Compress file. * @param src source filename @@ -181,6 +184,8 @@ gboolean cr_copy_file(const char *src, * If dst is NULL, src + compression suffix is used) * @param comtype type of compression * @param stat pointer to cr_ContentStat or NULL + * @param zck_dict_dir Location of zchunk zdicts (if zchunk is enabled) + * @param zck_auto_chunk Whether zchunk file should be auto-chunked * @param err GError ** * @return cr_Error return code */ @@ -188,6 +193,8 @@ int cr_compress_file_with_stat(const char *src, const char *dst, cr_CompressionType comtype, cr_ContentStat *stat, + const char *zck_dict_dir, + gboolean zck_auto_chunk, GError **err); /** Decompress file. @@ -541,7 +548,7 @@ cr_version_string_with_features(void); * exists or no dictionary directory is set, this function returns NULL * * The zchunk dictionary file must be the same as the passed filename with a - * ".dict" extension + * ".zdict" extension * * @param dir Zchunk dictionary directory * @param file File being zchunked diff --git a/src/modifyrepo_c.c b/src/modifyrepo_c.c index 649d8b3d..dc8954fe 100644 --- a/src/modifyrepo_c.c +++ b/src/modifyrepo_c.c @@ -54,6 +54,8 @@ typedef struct { gboolean verbose; gchar *batchfile; gchar *new_name; + gboolean zck; + gchar *zck_dict_dir; } RawCmdOptions; @@ -92,6 +94,12 @@ parse_arguments(int *argc, char ***argv, RawCmdOptions *options, GError **err) "Batch file.", "BATCHFILE" }, { "new-name", 0, 0, G_OPTION_ARG_STRING, &(options->new_name), "New filename for the file", "NEWFILENAME"}, +#ifdef WITH_ZCHUNK + { "zck", 0, 0, G_OPTION_ARG_NONE, &(options->zck), + "Generate zchunk files as well as the standard repodata.", NULL }, + { "zck-dict-dir", 0, 0, G_OPTION_ARG_FILENAME, &(options->zck_dict_dir), + "Directory containing compression dictionaries for use by zchunk", "ZCK_DICT_DIR" }, +#endif { NULL, 0, 0, G_OPTION_ARG_NONE, NULL, NULL, NULL }, }; @@ -109,6 +117,8 @@ parse_arguments(int *argc, char ***argv, RawCmdOptions *options, GError **err) options->verbose = FALSE; options->batchfile = NULL; options->new_name = NULL; + options->zck = FALSE; + options->zck_dict_dir = NULL; GOptionContext *context; context = g_option_context_new(" \n" @@ -165,6 +175,15 @@ check_arguments(RawCmdOptions *options, GError **err) return FALSE; } + // Zchunk options + if(options->zck_dict_dir && !options->zck) { + g_set_error(err, ERR_DOMAIN, CRE_ERROR, + "Cannot use --zck-dict-dir without setting --zck"); + return FALSE; + } + if (options->zck_dict_dir) + options->zck_dict_dir = cr_normalize_dir_path(options->zck_dict_dir); + return TRUE; } @@ -217,6 +236,8 @@ cmd_options_to_task(GSList **modifyrepotasks, task->checksum_type = cr_checksum_type(options->checksum); task->new_name = cr_safe_string_chunk_insert_null(task->chunk, options->new_name); + task->zck = options->zck; + task->zck_dict_dir = options->zck_dict_dir; *modifyrepotasks = g_slist_append(*modifyrepotasks, task); diff --git a/src/modifyrepo_shared.c b/src/modifyrepo_shared.c index 805c8943..a84d29d6 100644 --- a/src/modifyrepo_shared.c +++ b/src/modifyrepo_shared.c @@ -50,6 +50,64 @@ cr_modifyrepotask_free(cr_ModifyRepoTask *task) g_free(task); } +static gchar * +write_file(gchar *repopath, cr_ModifyRepoTask *task, + cr_CompressionType compress_type, GError **err) +{ + const gchar *suffix = NULL; + + if (task->compress) + suffix = cr_compression_suffix(compress_type); + + gchar *src_fn = task->path; // Shortcut + gchar *dst_fn = NULL; + + // Prepare dst filename - Get basename + _cleanup_free_ gchar *filename = NULL; + if (task->new_name) + filename = g_path_get_basename(task->new_name); + else + filename = g_path_get_basename(src_fn); + + // Prepare dst filename - Add suffix + if (suffix) { + gchar *tmp_fn = g_strconcat(filename, suffix, NULL); + g_free(filename); + filename = tmp_fn; + } + + // Prepare dst filename - Full path + dst_fn = g_build_filename(repopath, filename, NULL); + task->dst_fn = g_string_chunk_insert(task->chunk, dst_fn); + + // Check if the src and dst is the same file + gboolean identical = FALSE; + if (!cr_identical_files(src_fn, dst_fn, &identical, err)) + return NULL; + + if (identical) { + // Source and destination file is the same file + g_debug("Using already existing file: %s", dst_fn); + } else { + // Check if the file already exist + if (g_file_test(dst_fn, G_FILE_TEST_EXISTS)) { + g_warning("Destination file \"%s\" already exists and will be " + "overwritten", dst_fn); + } + + // Do the copy + g_debug("%s: Copy & compress operation %s -> %s", + __func__, src_fn, dst_fn); + + if (cr_compress_file(src_fn, dst_fn, compress_type, + task->zck_dict_dir, TRUE, err) != CRE_OK) { + g_debug("%s: Copy & compress operation failed", __func__); + return NULL; + } + } + return dst_fn; +} + gboolean cr_modifyrepo(GSList *modifyrepotasks, gchar *repopath, GError **err) { @@ -177,67 +235,37 @@ cr_modifyrepo(GSList *modifyrepotasks, gchar *repopath, GError **err) // Add (copy) new metadata to repodata/ directory for (GSList *elem = modifyrepotasks; elem; elem = g_slist_next(elem)) { cr_ModifyRepoTask *task = elem->data; + _cleanup_free_ gchar *dst_fn = NULL; if (task->remove) // Skip removing task continue; - gchar *src_fn = task->path; // Shortcut - _cleanup_free_ gchar *dst_fn = NULL; - const gchar *suffix = NULL; cr_CompressionType compress_type = CR_CW_NO_COMPRESSION; - if (task->compress) { + if (task->compress) compress_type = task->compress_type; - suffix = cr_compression_suffix(compress_type); - } - // Prepare dst filename - Get basename - _cleanup_free_ gchar *filename = NULL; - if (task->new_name) - filename = g_path_get_basename(task->new_name); - else - filename = g_path_get_basename(src_fn); - - // Prepare dst filename - Add suffix - if (suffix) { - gchar *tmp_fn = g_strconcat(filename, suffix, NULL); - g_free(filename); - filename = tmp_fn; - } - - // Prepare dst filename - Full path - dst_fn = g_build_filename(repopath, filename, NULL); - task->dst_fn = g_string_chunk_insert(task->chunk, dst_fn); - - // Check if the src and dst is the same file - gboolean identical = FALSE; - if (!cr_identical_files(src_fn, dst_fn, &identical, err)) + dst_fn = write_file(repopath, task, compress_type, err); + if(dst_fn == NULL) { + cr_repomd_free(repomd); + g_free(repomd_path); return FALSE; - - if (identical) { - // Source and destination file is the same file - g_debug("Using already existing file: %s", dst_fn); - } else { - // Check if the file already exist - if (g_file_test(dst_fn, G_FILE_TEST_EXISTS)) { - g_warning("Destination file \"%s\" already exists and will be " - "overwritten", dst_fn); - } - - // Do the copy - g_debug("%s: Copy & compress operation %s -> %s", - __func__, src_fn, dst_fn); - - if (cr_compress_file(src_fn, dst_fn, compress_type, err) != CRE_OK) { - g_debug("%s: Copy & compress operation failed", __func__); + } + + task->repopath = cr_safe_string_chunk_insert_null(task->chunk, dst_fn); +#ifdef WITH_ZCHUNK + if(task->zck) { + free(dst_fn); + dst_fn = write_file(repopath, task, CR_CW_ZCK_COMPRESSION, err); + if(dst_fn == NULL) { cr_repomd_free(repomd); g_free(repomd_path); return FALSE; } + task->zck_repopath = cr_safe_string_chunk_insert_null(task->chunk, dst_fn); } - - task->repopath = cr_safe_string_chunk_insert_null(task->chunk, dst_fn); +#endif } // Prepare new repomd records @@ -260,11 +288,25 @@ cr_modifyrepo(GSList *modifyrepotasks, gchar *repopath, GError **err) task->checksum_type, NULL); g_thread_pool_push(fill_pool, filltask, NULL); - repomdrecords = g_slist_append(repomdrecords, rec); + repomdrecords = g_slist_append(repomdrecords, rec); + if (task->unique_md_filenames) repomdrecords_uniquefn = g_slist_prepend(repomdrecords_uniquefn, rec); repomdrecordfilltasks = g_slist_prepend(repomdrecordfilltasks, filltask); + if(task->zck) { + _cleanup_free_ gchar *type = g_strconcat(task->type, "_zck", NULL); + rec = cr_repomd_record_new(type, task->zck_repopath); + filltask = cr_repomdrecordfilltask_new(rec, task->checksum_type, NULL); + g_thread_pool_push(fill_pool, filltask, NULL); + + repomdrecords = g_slist_append(repomdrecords, rec); + + if (task->unique_md_filenames) + repomdrecords_uniquefn = g_slist_prepend(repomdrecords_uniquefn, rec); + repomdrecordfilltasks = g_slist_prepend(repomdrecordfilltasks, + filltask); + } } g_thread_pool_free(fill_pool, FALSE, TRUE); // Wait @@ -289,6 +331,16 @@ cr_modifyrepo(GSList *modifyrepotasks, gchar *repopath, GError **err) __func__, task->type); recordstoremove = g_slist_prepend(recordstoremove, rec); cr_repomd_detach_record(repomd, rec); + if(task->zck) { + _cleanup_free_ gchar *type = g_strconcat(task->type, "_zck", NULL); + cr_RepomdRecord *rec = cr_repomd_get_record(repomd, type); + if (rec) { + g_debug("%s: Removing record \"%s\" from repomd.xml", + __func__, type); + recordstoremove = g_slist_prepend(recordstoremove, rec); + cr_repomd_detach_record(repomd, rec); + } + } } } diff --git a/src/modifyrepo_shared.h b/src/modifyrepo_shared.h index 1abbdc5a..a2853108 100644 --- a/src/modifyrepo_shared.h +++ b/src/modifyrepo_shared.h @@ -47,9 +47,12 @@ typedef struct { gboolean unique_md_filenames; cr_ChecksumType checksum_type; gchar *new_name; + gboolean zck; + gchar *zck_dict_dir; // Internal use gchar *repopath; + gchar *zck_repopath; gchar *dst_fn; GStringChunk *chunk; diff --git a/src/python/misc-py.c b/src/python/misc-py.c index 2582eddc..6a7871e6 100644 --- a/src/python/misc-py.c +++ b/src/python/misc-py.c @@ -49,7 +49,7 @@ py_compress_file_with_stat(G_GNUC_UNUSED PyObject *self, PyObject *args) return NULL; } - cr_compress_file_with_stat(src, dst, type, contentstat, &tmp_err); + cr_compress_file_with_stat(src, dst, type, contentstat, NULL, FALSE, &tmp_err); if (tmp_err) { nice_exception(&tmp_err, NULL); return NULL; diff --git a/src/python/repomdrecord-py.c b/src/python/repomdrecord-py.c index 34c86d03..1bbd6b9b 100644 --- a/src/python/repomdrecord-py.c +++ b/src/python/repomdrecord-py.c @@ -182,13 +182,15 @@ compress_and_fill(_RepomdRecordObject *self, PyObject *args) { int checksum_type, compression_type; PyObject *compressed_repomdrecord; + gchar *zck_dict_dir = NULL; GError *err = NULL; - if (!PyArg_ParseTuple(args, "O!ii:compress_and_fill", + if (!PyArg_ParseTuple(args, "O!ii|s:compress_and_fill", &RepomdRecord_Type, &compressed_repomdrecord, &checksum_type, - &compression_type)) + &compression_type, + &zck_dict_dir)) return NULL; if (check_RepomdRecordStatus(self)) @@ -198,6 +200,7 @@ compress_and_fill(_RepomdRecordObject *self, PyObject *args) RepomdRecord_FromPyObject(compressed_repomdrecord), checksum_type, compression_type, + zck_dict_dir, &err); if (err) { nice_exception(&err, NULL); diff --git a/src/repomd.c b/src/repomd.c index 6dbfbf42..cf69610a 100644 --- a/src/repomd.c +++ b/src/repomd.c @@ -26,6 +26,7 @@ #include #include #include +#include "cleanup.h" #include "error.h" #include "misc.h" #include "checksum.h" @@ -340,6 +341,7 @@ cr_repomd_record_compress_and_fill(cr_RepomdRecord *record, cr_RepomdRecord *crecord, cr_ChecksumType checksum_type, cr_CompressionType record_compression, + const char *zck_dict_dir, GError **err) { int ret = CRE_OK; @@ -409,6 +411,27 @@ cr_repomd_record_compress_and_fill(cr_RepomdRecord *record, return ret; } + _cleanup_free_ gchar *dict = NULL; + size_t dict_size = 0; + if(record_compression == CR_CW_ZCK_COMPRESSION && zck_dict_dir) { + /* Find zdict */ + _cleanup_free_ gchar *file_basename = NULL; + _cleanup_free_ gchar *dict_base = NULL; + if(g_str_has_suffix(cpath, ".zck")) + dict_base = g_strndup(cpath, strlen(cpath)-4); + else + dict_base = g_strdup(cpath); + file_basename = g_path_get_basename(dict_base); + _cleanup_free_ gchar *dict_file = cr_get_dict_file(zck_dict_dir, file_basename); + /* Read dictionary from file */ + if(dict_file && !g_file_get_contents(dict_file, &dict, + &dict_size, &tmp_err)) { + ret = tmp_err->code; + g_propagate_prefixed_error(err, tmp_err, "Error reading zchunk dict %s:", dict_file); + return ret; + } + } + cw_compressed = cr_open(cpath, CR_CW_MODE_WRITE, record_compression, @@ -419,6 +442,19 @@ cr_repomd_record_compress_and_fill(cr_RepomdRecord *record, return ret; } + if(record_compression == CR_CW_ZCK_COMPRESSION) { + if(dict && cr_set_dict(cw_compressed, dict, dict_size, &tmp_err) != CRE_OK) { + ret = tmp_err->code; + g_propagate_prefixed_error(err, tmp_err, "Unable to set zdict for %s: ", cpath); + return ret; + } + if(cr_set_autochunk(cw_compressed, TRUE, &tmp_err) != CRE_OK) { + ret = tmp_err->code; + g_propagate_prefixed_error(err, tmp_err, "Unable to set auto-chunking for %s: ", cpath); + return ret; + } + } + while ((readed = cr_read(cw_plain, buf, BUFFER_SIZE, &tmp_err)) > 0) { cr_write(cw_compressed, buf, (unsigned int) readed, &tmp_err); if (tmp_err) diff --git a/src/repomd.h b/src/repomd.h index 764e1d76..0ad3c741 100644 --- a/src/repomd.h +++ b/src/repomd.h @@ -159,6 +159,7 @@ int cr_repomd_record_fill(cr_RepomdRecord *record, * @param compressed_record empty cr_RepomdRecord object that will by filled * @param checksum_type type of checksums * @param compression type of compression + * @param zck_dict_dir Location of zchunk dictionaries (NULL if unused) * @param err GError ** * @return cr_Error code */ @@ -166,6 +167,7 @@ int cr_repomd_record_compress_and_fill(cr_RepomdRecord *record, cr_RepomdRecord *compressed_record, cr_ChecksumType checksum_type, cr_CompressionType compression, + const char *zck_dict_dir, GError **err); /** Add a hash as prefix to the filename. diff --git a/src/sqliterepo_c.c b/src/sqliterepo_c.c index f147a651..0899ebde 100644 --- a/src/sqliterepo_c.c +++ b/src/sqliterepo_c.c @@ -391,21 +391,21 @@ compress_sqlite_dbs(const gchar *tmp_out_repo, pri_db_name, compression_type, checksum_type, - 1, NULL); + NULL, FALSE, 1, NULL); g_thread_pool_push(compress_pool, pri_db_task, NULL); fil_db_task = cr_compressiontask_new(fil_db_filename, fil_db_name, compression_type, checksum_type, - 1, NULL); + NULL, FALSE, 1, NULL); g_thread_pool_push(compress_pool, fil_db_task, NULL); oth_db_task = cr_compressiontask_new(oth_db_filename, oth_db_name, compression_type, checksum_type, - 1, NULL); + NULL, FALSE, 1, NULL); g_thread_pool_push(compress_pool, oth_db_task, NULL); // Wait till all tasks are complete and free the thread pool diff --git a/src/threads.c b/src/threads.c index aee07d13..a53e79b0 100644 --- a/src/threads.c +++ b/src/threads.c @@ -32,6 +32,8 @@ cr_compressiontask_new(const char *src, const char *dst, cr_CompressionType compression_type, cr_ChecksumType checksum_type, + const char *zck_dict_dir, + gboolean zck_auto_chunk, int delsrc, GError **err) { @@ -58,6 +60,9 @@ cr_compressiontask_new(const char *src, task->dst = g_strdup(dst); task->type = compression_type; task->stat = stat; + if(zck_dict_dir != NULL) + task->zck_dict_dir = g_strdup(zck_dict_dir); + task->zck_auto_chunk = zck_auto_chunk; task->delsrc = delsrc; return task; @@ -76,6 +81,8 @@ cr_compressiontask_free(cr_CompressionTask *task, GError **err) cr_contentstat_free(task->stat, err); if (task->err) g_error_free(task->err); + if (task->zck_dict_dir) + g_free(task->zck_dict_dir); g_free(task); } @@ -96,6 +103,8 @@ cr_compressing_thread(gpointer data, G_GNUC_UNUSED gpointer user_data) task->dst, task->type, task->stat, + task->zck_dict_dir, + task->zck_auto_chunk, &tmp_err); if (tmp_err) { diff --git a/src/threads.h b/src/threads.h index 2d554cdc..af72832e 100644 --- a/src/threads.h +++ b/src/threads.h @@ -76,6 +76,10 @@ typedef struct { Type of compression to use */ cr_ContentStat *stat; /*!< Stats of compressed file or NULL */ + char *zck_dict_dir; /*!< + Location of zchunk dictionaries */ + gboolean zck_auto_chunk; /*!< + Whether zchunk file should be auto-chunked */ int delsrc; /*!< Indicate if delete source file after successful compression. */ GError *err; /*!< @@ -106,6 +110,8 @@ cr_compressiontask_new(const char *src, const char *dst, cr_CompressionType compression_type, cr_ChecksumType checksum_type, + const char *zck_dict_dir, + gboolean zck_auto_chunk, int delsrc, GError **err); From 61d2dce30dd0f2a72e3354e850513eabc3c18310 Mon Sep 17 00:00:00 2001 From: Jonathan Dieter Date: Tue, 15 Jan 2019 23:10:06 +0000 Subject: [PATCH 3/6] Add zchunk support to mergerepo_c for extra files Signed-off-by: Jonathan Dieter --- src/mergerepo_c.c | 33 ++++++++++++++ src/misc.c | 95 ++++++++++++++++++++++------------------- src/misc.h | 4 +- src/modifyrepo_shared.c | 5 ++- src/python/misc-py.c | 2 +- src/repomd.c | 26 +++++++---- src/threads.c | 2 +- 7 files changed, 108 insertions(+), 59 deletions(-) diff --git a/src/mergerepo_c.c b/src/mergerepo_c.c index fbdde5a5..bd6dcee1 100644 --- a/src/mergerepo_c.c +++ b/src/mergerepo_c.c @@ -1509,8 +1509,11 @@ dump_merged_metadata(GHashTable *merged_hashtable, cr_RepomdRecord *oth_zck_rec = NULL; cr_RepomdRecord *groupfile_rec = NULL; cr_RepomdRecord *compressed_groupfile_rec = NULL; + cr_RepomdRecord *groupfile_zck_rec = NULL; cr_RepomdRecord *update_info_rec = NULL; + cr_RepomdRecord *update_info_zck_rec = NULL; cr_RepomdRecord *pkgorigins_rec = NULL; + cr_RepomdRecord *pkgorigins_zck_rec = NULL; // XML @@ -1555,6 +1558,14 @@ dump_merged_metadata(GHashTable *merged_hashtable, CR_CHECKSUM_SHA256, cmd_options->groupfile_compression_type, NULL, NULL); + if(cmd_options->zck_compression) { + groupfile_zck_rec = cr_repomd_record_new("group_zck", groupfile); + cr_repomd_record_compress_and_fill(groupfile_rec, + groupfile_zck_rec, + CR_CHECKSUM_SHA256, + CR_CW_ZCK_COMPRESSION, + NULL, NULL); + } } @@ -1563,6 +1574,14 @@ dump_merged_metadata(GHashTable *merged_hashtable, if (!cmd_options->noupdateinfo) { update_info_rec = cr_repomd_record_new("updateinfo", update_info_filename); cr_repomd_record_fill(update_info_rec, CR_CHECKSUM_SHA256, NULL); + if(cmd_options->zck_compression) { + update_info_zck_rec = cr_repomd_record_new("updateinfo_zck", update_info_filename); + cr_repomd_record_compress_and_fill(update_info_rec, + update_info_zck_rec, + CR_CHECKSUM_SHA256, + CR_CW_ZCK_COMPRESSION, + NULL, NULL); + } } @@ -1572,6 +1591,14 @@ dump_merged_metadata(GHashTable *merged_hashtable, gchar *pkgorigins_path = g_strconcat(cmd_options->tmp_out_repo, "pkgorigins.gz", NULL); pkgorigins_rec = cr_repomd_record_new("origin", pkgorigins_path); cr_repomd_record_fill(pkgorigins_rec, CR_CHECKSUM_SHA256, NULL); + if(cmd_options->zck_compression) { + pkgorigins_zck_rec = cr_repomd_record_new("origin_zck", pkgorigins_path); + cr_repomd_record_compress_and_fill(pkgorigins_rec, + pkgorigins_zck_rec, + CR_CHECKSUM_SHA256, + CR_CW_ZCK_COMPRESSION, + NULL, NULL); + } g_free(pkgorigins_path); } @@ -1752,8 +1779,11 @@ dump_merged_metadata(GHashTable *merged_hashtable, cr_repomd_record_rename_file(oth_zck_rec, NULL); cr_repomd_record_rename_file(groupfile_rec, NULL); cr_repomd_record_rename_file(compressed_groupfile_rec, NULL); + cr_repomd_record_rename_file(groupfile_zck_rec, NULL); cr_repomd_record_rename_file(update_info_rec, NULL); + cr_repomd_record_rename_file(update_info_zck_rec, NULL); cr_repomd_record_rename_file(pkgorigins_rec, NULL); + cr_repomd_record_rename_file(pkgorigins_zck_rec, NULL); } @@ -1771,8 +1801,11 @@ dump_merged_metadata(GHashTable *merged_hashtable, cr_repomd_set_record(repomd_obj, oth_zck_rec); cr_repomd_set_record(repomd_obj, groupfile_rec); cr_repomd_set_record(repomd_obj, compressed_groupfile_rec); + cr_repomd_set_record(repomd_obj, groupfile_zck_rec); cr_repomd_set_record(repomd_obj, update_info_rec); + cr_repomd_set_record(repomd_obj, update_info_zck_rec); cr_repomd_set_record(repomd_obj, pkgorigins_rec); + cr_repomd_set_record(repomd_obj, pkgorigins_zck_rec); char *repomd_xml = cr_xml_dump_repomd(repomd_obj, NULL); diff --git a/src/misc.c b/src/misc.c index e9a5568b..689c076f 100644 --- a/src/misc.c +++ b/src/misc.c @@ -429,7 +429,7 @@ cr_copy_file(const char *src, const char *in_dst, GError **err) int cr_compress_file_with_stat(const char *src, - const char *in_dst, + char **in_dst, cr_CompressionType compression, cr_ContentStat *stat, const char *zck_dict_dir, @@ -439,14 +439,16 @@ cr_compress_file_with_stat(const char *src, int ret = CRE_OK; int readed; char buf[BUFFER_SIZE]; - FILE *orig = NULL; + CR_FILE *orig = NULL; CR_FILE *new = NULL; - gchar *dst = (gchar *) in_dst; + gchar *dst = (gchar *) *in_dst; GError *tmp_err = NULL; assert(src); assert(!err || *err == NULL); + const char *c_suffix = cr_compression_suffix(compression); + // Src must be a file NOT a directory if (!g_file_test(src, G_FILE_TEST_IS_REGULAR)) { g_debug("%s: Source (%s) must be a regular file!", __func__, src); @@ -457,25 +459,43 @@ cr_compress_file_with_stat(const char *src, if (!dst) { // If destination is NULL, use src + compression suffix - dst = g_strconcat(src, - cr_compression_suffix(compression), - NULL); - } else if (g_str_has_suffix(in_dst, "/")) { + *in_dst = g_strconcat(src, + c_suffix, + NULL); + } else if (g_str_has_suffix(dst, "/")) { // If destination is dir use filename from src + compression suffix - dst = g_strconcat(in_dst, - cr_get_filename(src), - cr_compression_suffix(compression), - NULL); + *in_dst = g_strconcat(dst, + cr_get_filename(src), + c_suffix, + NULL); + } else if (c_suffix && !g_str_has_suffix(dst, c_suffix)) { + cr_CompressionType old_type = cr_detect_compression(src, &tmp_err); + if(tmp_err) { + g_debug("%s: Unable to detect compression type of %s", __func__, src); + g_clear_error(&tmp_err); + } else if(old_type != CR_CW_NO_COMPRESSION) { + _cleanup_free_ gchar *tmp_file = g_strndup(dst, strlen(dst) - strlen(cr_compression_suffix(old_type))); + *in_dst = g_strconcat(tmp_file, + c_suffix, + NULL); + } } - - orig = fopen(src, "rb"); - if (orig == NULL) { - g_debug("%s: Cannot open source file %s (%s)", __func__, src, - g_strerror(errno)); - g_set_error(err, ERR_DOMAIN, CRE_IO, - "Cannot open %s: %s", src, g_strerror(errno)); - ret = CRE_IO; - goto compress_file_cleanup; + if (dst != *in_dst && dst) + g_free(dst); + dst = (gchar *) *in_dst; + + int mode = CR_CW_NO_COMPRESSION; + if(compression == CR_CW_ZCK_COMPRESSION) + mode = CR_CW_AUTO_DETECT_COMPRESSION; + + orig = cr_open(src, + CR_CW_MODE_READ, + mode, + &tmp_err); + if (!orig) { + ret = tmp_err->code; + g_propagate_prefixed_error(err, tmp_err, "Cannot open %s: ", src); + return ret; } _cleanup_free_ gchar *dict = NULL; @@ -483,12 +503,12 @@ cr_compress_file_with_stat(const char *src, if(compression == CR_CW_ZCK_COMPRESSION && zck_dict_dir) { /* Find zdict */ _cleanup_free_ gchar *file_basename = NULL; - if(in_dst) { + if(dst) { _cleanup_free_ gchar *dict_base = NULL; - if(g_str_has_suffix(in_dst, ".zck")) - dict_base = g_strndup(in_dst, strlen(in_dst)-4); + if(g_str_has_suffix(dst, ".zck")) + dict_base = g_strndup(dst, strlen(dst)-4); else - dict_base = g_strdup(in_dst); + dict_base = g_strdup(dst); file_basename = g_path_get_basename(dict_base); } else { file_basename = g_path_get_basename(src); @@ -517,42 +537,27 @@ cr_compress_file_with_stat(const char *src, if(dict && cr_set_dict(new, dict, dict_size, &tmp_err) != CRE_OK) { ret = tmp_err->code; g_propagate_prefixed_error(err, tmp_err, "Unable to set zdict for %s: ", dst); - return ret; + goto compress_file_cleanup; } if(zck_auto_chunk && cr_set_autochunk(new, TRUE, &tmp_err) != CRE_OK) { ret = tmp_err->code; g_propagate_prefixed_error(err, tmp_err, "Unable to set auto-chunking for %s: ", dst); - return ret; - } - } - - while ((readed = fread(buf, 1, BUFFER_SIZE, orig)) > 0) { - if (readed != BUFFER_SIZE && ferror(orig)) { - g_debug("%s: Error while copy %s -> %s (%s)", __func__, src, - dst, g_strerror(errno)); - g_set_error(err, ERR_DOMAIN, CRE_IO, - "Error while read %s: %s", src, g_strerror(errno)); - ret = CRE_IO; goto compress_file_cleanup; } - + } + while ((readed = cr_read(orig, buf, BUFFER_SIZE, &tmp_err)) > 0) { cr_write(new, buf, readed, &tmp_err); if (tmp_err) { - g_debug("%s: Error while copy %s -> %s", __func__, src, dst); - g_propagate_prefixed_error(err, tmp_err, - "Error while read %s: ", dst); - ret = CRE_IO; + ret = tmp_err->code; + g_propagate_prefixed_error(err, tmp_err, "Unable to write to %s: ", dst); goto compress_file_cleanup; } } compress_file_cleanup: - if (dst != in_dst) - g_free(dst); - if (orig) - fclose(orig); + cr_close(orig, NULL); if (new) cr_close(new, NULL); diff --git a/src/misc.h b/src/misc.h index e6dd0278..b462f2a0 100644 --- a/src/misc.h +++ b/src/misc.h @@ -179,7 +179,7 @@ gboolean cr_copy_file(const char *src, /** Compress file. * @param src source filename - * @param dst destination (If dst is dir, filename of src + + * @param dst pointer to destination (If dst is dir, filename of src + * compression suffix is used. * If dst is NULL, src + compression suffix is used) * @param comtype type of compression @@ -190,7 +190,7 @@ gboolean cr_copy_file(const char *src, * @return cr_Error return code */ int cr_compress_file_with_stat(const char *src, - const char *dst, + char **dst, cr_CompressionType comtype, cr_ContentStat *stat, const char *zck_dict_dir, diff --git a/src/modifyrepo_shared.c b/src/modifyrepo_shared.c index a84d29d6..93a8965d 100644 --- a/src/modifyrepo_shared.c +++ b/src/modifyrepo_shared.c @@ -90,7 +90,8 @@ write_file(gchar *repopath, cr_ModifyRepoTask *task, g_debug("Using already existing file: %s", dst_fn); } else { // Check if the file already exist - if (g_file_test(dst_fn, G_FILE_TEST_EXISTS)) { + if (g_file_test(dst_fn, G_FILE_TEST_EXISTS) && + g_str_has_suffix(dst_fn, cr_compression_suffix(compress_type))) { g_warning("Destination file \"%s\" already exists and will be " "overwritten", dst_fn); } @@ -99,7 +100,7 @@ write_file(gchar *repopath, cr_ModifyRepoTask *task, g_debug("%s: Copy & compress operation %s -> %s", __func__, src_fn, dst_fn); - if (cr_compress_file(src_fn, dst_fn, compress_type, + if (cr_compress_file(src_fn, &dst_fn, compress_type, task->zck_dict_dir, TRUE, err) != CRE_OK) { g_debug("%s: Copy & compress operation failed", __func__); return NULL; diff --git a/src/python/misc-py.c b/src/python/misc-py.c index 6a7871e6..5daff489 100644 --- a/src/python/misc-py.c +++ b/src/python/misc-py.c @@ -49,7 +49,7 @@ py_compress_file_with_stat(G_GNUC_UNUSED PyObject *self, PyObject *args) return NULL; } - cr_compress_file_with_stat(src, dst, type, contentstat, NULL, FALSE, &tmp_err); + cr_compress_file_with_stat(src, &dst, type, contentstat, NULL, FALSE, &tmp_err); if (tmp_err) { nice_exception(&tmp_err, NULL); return NULL; diff --git a/src/repomd.c b/src/repomd.c index cf69610a..cb11696d 100644 --- a/src/repomd.c +++ b/src/repomd.c @@ -357,7 +357,7 @@ cr_repomd_record_compress_and_fill(cr_RepomdRecord *record, CR_FILE *cw_compressed; gint64 gf_size = G_GINT64_CONSTANT(-1), cgf_size = G_GINT64_CONSTANT(-1); gint64 gf_time = G_GINT64_CONSTANT(-1), cgf_time = G_GINT64_CONSTANT(-1); - gint64 gf_hdrsize = G_GINT64_CONSTANT(-1); + gint64 cgf_hdrsize = G_GINT64_CONSTANT(-1); struct stat gf_stat, cgf_stat; const char *checksum_str = cr_checksum_name_str(checksum_type); const char *hdr_checksum_str = NULL; @@ -401,9 +401,13 @@ cr_repomd_record_compress_and_fill(cr_RepomdRecord *record, // Compress file + get size of non compressed file + int mode = CR_CW_NO_COMPRESSION; + if(record_compression == CR_CW_ZCK_COMPRESSION) + mode = CR_CW_AUTO_DETECT_COMPRESSION; + cw_plain = cr_open(path, CR_CW_MODE_READ, - CR_CW_NO_COMPRESSION, + mode, &tmp_err); if (!cw_plain) { ret = tmp_err->code; @@ -432,10 +436,12 @@ cr_repomd_record_compress_and_fill(cr_RepomdRecord *record, } } - cw_compressed = cr_open(cpath, - CR_CW_MODE_WRITE, - record_compression, - &tmp_err); + _cleanup_free_ cr_ContentStat *out_stat = g_malloc0(sizeof(cr_ContentStat)); + cw_compressed = cr_sopen(cpath, + CR_CW_MODE_WRITE, + record_compression, + out_stat, + &tmp_err); if (!cw_compressed) { ret = tmp_err->code; g_propagate_prefixed_error(err, tmp_err, "Cannot open %s: ", cpath); @@ -523,7 +529,11 @@ cr_repomd_record_compress_and_fill(cr_RepomdRecord *record, cgf_size = cgf_stat.st_size; cgf_time = cgf_stat.st_mtime; - + if(out_stat->hdr_checksum) { + cgf_hdrsize = out_stat->hdr_size; + hdr_checksum_str = cr_checksum_name_str(out_stat->hdr_checksum_type); + hdrchecksum = out_stat->hdr_checksum; + } // Results @@ -552,7 +562,7 @@ cr_repomd_record_compress_and_fill(cr_RepomdRecord *record, crecord->timestamp = cgf_time; crecord->size = cgf_size; crecord->size_open = gf_size; - crecord->size_header = gf_hdrsize; + crecord->size_header = cgf_hdrsize; end: g_free(checksum); diff --git a/src/threads.c b/src/threads.c index a53e79b0..54f8416f 100644 --- a/src/threads.c +++ b/src/threads.c @@ -100,7 +100,7 @@ cr_compressing_thread(gpointer data, G_GNUC_UNUSED gpointer user_data) NULL); cr_compress_file_with_stat(task->src, - task->dst, + &(task->dst), task->type, task->stat, task->zck_dict_dir, From fed36d5f2d686574be247fe60747cc097fb3ce85 Mon Sep 17 00:00:00 2001 From: Jonathan Dieter Date: Tue, 15 Jan 2019 23:23:47 +0000 Subject: [PATCH 4/6] Add missing sentinal Signed-off-by: Jonathan Dieter --- src/misc.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/misc.c b/src/misc.c index 689c076f..710782a4 100644 --- a/src/misc.c +++ b/src/misc.c @@ -1566,7 +1566,7 @@ cr_get_dict_file(const gchar *dir, const gchar *file) assert(dict_file); snprintf(dict_file, strlen(file) + 7, "%s.zdict", file); - gchar *full_path = g_build_path("/", dir, dict_file); + gchar *full_path = g_build_path("/", dir, dict_file, NULL); assert(full_path); free(dict_file); From 5e3aa71eac6f02049fbeff6492fd7a3d1fc70a38 Mon Sep 17 00:00:00 2001 From: Jonathan Dieter Date: Tue, 15 Jan 2019 23:23:58 +0000 Subject: [PATCH 5/6] Fix misc test Signed-off-by: Jonathan Dieter --- tests/test_misc.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tests/test_misc.c b/tests/test_misc.c index 9d528c70..06144a6d 100644 --- a/tests/test_misc.c +++ b/tests/test_misc.c @@ -531,8 +531,8 @@ compressfile_test_text_file(Copyfiletest *copyfiletest, GError *tmp_err = NULL; g_assert(!g_file_test(copyfiletest->dst_file, G_FILE_TEST_EXISTS)); - ret = cr_compress_file(TEST_TEXT_FILE, copyfiletest->dst_file, - CR_CW_GZ_COMPRESSION, &tmp_err); + ret = cr_compress_file(TEST_TEXT_FILE, &(copyfiletest->dst_file), + CR_CW_GZ_COMPRESSION, NULL, FALSE, &tmp_err); g_assert(!tmp_err); g_assert_cmpint(ret, ==, CRE_OK); g_assert(g_file_test(copyfiletest->dst_file, G_FILE_TEST_IS_REGULAR)); @@ -556,8 +556,9 @@ compressfile_with_stat_test_text_file(Copyfiletest *copyfiletest, g_assert(!tmp_err); g_assert(!g_file_test(copyfiletest->dst_file, G_FILE_TEST_EXISTS)); - ret = cr_compress_file_with_stat(TEST_TEXT_FILE, copyfiletest->dst_file, - CR_CW_GZ_COMPRESSION, stat, &tmp_err); + ret = cr_compress_file_with_stat(TEST_TEXT_FILE, ©filetest->dst_file, + CR_CW_GZ_COMPRESSION, stat, NULL, FALSE, + &tmp_err); g_assert(!tmp_err); g_assert_cmpint(ret, ==, CRE_OK); g_assert(g_file_test(copyfiletest->dst_file, G_FILE_TEST_IS_REGULAR)); From 0b2080c69343609e4561f7a368c3b74d449c99f3 Mon Sep 17 00:00:00 2001 From: Jonathan Dieter Date: Wed, 16 Jan 2019 19:07:30 +0000 Subject: [PATCH 6/6] Add space after if statement Signed-off-by: Jonathan Dieter --- src/cmd_parser.c | 2 +- src/compression_wrapper.c | 40 +++++++++++++++++++-------------------- src/createrepo_c.c | 20 ++++++++++---------- src/deltarpms.c | 2 +- src/dumper_thread.c | 10 +++++----- src/locate_metadata.c | 2 +- src/mergerepo_c.c | 28 +++++++++++++-------------- src/misc.c | 22 ++++++++++----------- src/modifyrepo_c.c | 2 +- src/modifyrepo_shared.c | 10 +++++----- src/repomd.c | 28 +++++++++++++-------------- src/threads.c | 2 +- 12 files changed, 84 insertions(+), 84 deletions(-) diff --git a/src/cmd_parser.c b/src/cmd_parser.c index 40d28c4f..2fb7696b 100644 --- a/src/cmd_parser.c +++ b/src/cmd_parser.c @@ -564,7 +564,7 @@ check_arguments(struct CmdOptions *options, } // Zchunk options - if(options->zck_dict_dir && !options->zck_compression) { + if (options->zck_dict_dir && !options->zck_compression) { g_set_error(err, ERR_DOMAIN, CRE_BADARG, "Cannot use --zck-dict-dir without setting --zck"); return FALSE; diff --git a/src/compression_wrapper.c b/src/compression_wrapper.c index 8dbb25b6..2526d607 100644 --- a/src/compression_wrapper.c +++ b/src/compression_wrapper.c @@ -296,18 +296,18 @@ cr_ChecksumType cr_cktype_from_zck(zckCtx *zck, GError **err) { int cktype = zck_get_full_hash_type(zck); - if(cktype < 0) { + if (cktype < 0) { g_set_error(err, ERR_DOMAIN, CRE_ZCK, "Unable to read hash from zchunk file"); return CR_CHECKSUM_UNKNOWN; } - if(cktype == ZCK_HASH_SHA1) + if (cktype == ZCK_HASH_SHA1) return CR_CHECKSUM_SHA1; - else if(cktype == ZCK_HASH_SHA256) + else if (cktype == ZCK_HASH_SHA256) return CR_CHECKSUM_SHA256; else { const char *ckname = zck_hash_name_from_type(cktype); - if(ckname == NULL) + if (ckname == NULL) ckname = "Unknown"; g_set_error(err, ERR_DOMAIN, CRE_ZCK, "Unknown zchunk checksum type: %s", ckname); @@ -600,7 +600,7 @@ cr_sopen(const char *filename, file->FILE = (void *) zck_create(); zckCtx *zck = file->FILE; if (mode == CR_CW_MODE_WRITE) { - if(!file->FILE || !zck_init_write(zck, fd) || + if (!file->FILE || !zck_init_write(zck, fd) || !zck_set_ioption(zck, ZCK_MANUAL_CHUNK, 1)) { zck_set_log_fd(STDOUT_FILENO); g_set_error(err, ERR_DOMAIN, CRE_IO, "%s", @@ -654,10 +654,10 @@ cr_sopen(const char *filename, #ifdef WITH_ZCHUNK /* Fill zchunk header_stat with header information */ - if(mode == CR_CW_MODE_READ && type == CR_CW_ZCK_COMPRESSION) { + if (mode == CR_CW_MODE_READ && type == CR_CW_ZCK_COMPRESSION) { zckCtx *zck = (zckCtx *)file->FILE; cr_ChecksumType cktype = cr_cktype_from_zck(zck, err); - if(cktype == CR_CHECKSUM_UNKNOWN) { + if (cktype == CR_CHECKSUM_UNKNOWN) { /* Error is already set in cr_cktype_from_zck */ g_free(file); return NULL; @@ -665,7 +665,7 @@ cr_sopen(const char *filename, file->stat->hdr_checksum_type = cktype; file->stat->hdr_checksum = zck_get_header_digest(zck); file->stat->hdr_size = zck_get_header_length(zck); - if(*err != NULL || file->stat->hdr_checksum == NULL || + if (*err != NULL || file->stat->hdr_checksum == NULL || file->stat->hdr_size < 0) { g_free(file); return NULL; @@ -685,7 +685,7 @@ cr_set_dict(CR_FILE *cr_file, const void *dict, unsigned int len, GError **err) int ret = CRE_OK; assert(!err || *err == NULL); - if(len == 0) + if (len == 0) return CRE_OK; switch (cr_file->type) { @@ -694,7 +694,7 @@ cr_set_dict(CR_FILE *cr_file, const void *dict, unsigned int len, GError **err) #ifdef WITH_ZCHUNK zckCtx *zck = (zckCtx *)cr_file->FILE; size_t wlen = (size_t)len; - if(!zck_set_soption(zck, ZCK_COMP_DICT, dict, wlen)) { + if (!zck_set_soption(zck, ZCK_COMP_DICT, dict, wlen)) { ret = CRE_ERROR; g_set_error(err, ERR_DOMAIN, CRE_ZCK, "Error setting dict"); @@ -815,7 +815,7 @@ cr_close(CR_FILE *cr_file, GError **err) rc = lzma_code(stream, LZMA_FINISH); - if(rc != LZMA_OK && rc != LZMA_STREAM_END) { + if (rc != LZMA_OK && rc != LZMA_STREAM_END) { // Error while coding const char *err_msg; @@ -860,7 +860,7 @@ cr_close(CR_FILE *cr_file, GError **err) break; } - if(rc == LZMA_STREAM_END) { + if (rc == LZMA_STREAM_END) { // Everything all right ret = CRE_OK; break; @@ -880,7 +880,7 @@ cr_close(CR_FILE *cr_file, GError **err) zckCtx *zck = (zckCtx *) cr_file->FILE; ret = CRE_OK; if (cr_file->mode == CR_CW_MODE_WRITE) { - if(zck_end_chunk(zck) < 0) { + if (zck_end_chunk(zck) < 0) { ret = CRE_ZCK; g_set_error(err, ERR_DOMAIN, CRE_ZCK, "Unable to end final chunk: %s", zck_get_error(zck)); @@ -892,15 +892,15 @@ cr_close(CR_FILE *cr_file, GError **err) "Unable to close zchunk file: %s", zck_get_error(zck)); } cr_ChecksumType cktype = cr_cktype_from_zck(zck, err); - if(cktype == CR_CHECKSUM_UNKNOWN) { + if (cktype == CR_CHECKSUM_UNKNOWN) { /* Error is already set in cr_cktype_from_zck */ break; } - if(cr_file->stat) { + if (cr_file->stat) { cr_file->stat->hdr_checksum_type = cktype; cr_file->stat->hdr_checksum = zck_get_header_digest(zck); cr_file->stat->hdr_size = zck_get_header_length(zck); - if((err && *err) || cr_file->stat->hdr_checksum == NULL || + if ((err && *err) || cr_file->stat->hdr_checksum == NULL || cr_file->stat->hdr_size < 0) { ret = CRE_ZCK; g_set_error(err, ERR_DOMAIN, CRE_ZCK, @@ -1121,7 +1121,7 @@ cr_read(CR_FILE *cr_file, void *buffer, unsigned int len, GError **err) #ifdef WITH_ZCHUNK zckCtx *zck = (zckCtx *) cr_file->FILE; ssize_t rb = zck_read(zck, buffer, len); - if(rb < 0) { + if (rb < 0) { ret = CR_CW_ERR; g_set_error(err, ERR_DOMAIN, CRE_ZCK, "ZCK: Unable to read: %s", zck_get_error(zck)); @@ -1307,7 +1307,7 @@ cr_write(CR_FILE *cr_file, const void *buffer, unsigned int len, GError **err) #ifdef WITH_ZCHUNK zckCtx *zck = (zckCtx *) cr_file->FILE; ssize_t wb = zck_write(zck, buffer, len); - if(wb < 0) { + if (wb < 0) { ret = CR_CW_ERR; g_set_error(err, ERR_DOMAIN, CRE_ZCK, "ZCK: Unable to write: %s", zck_get_error(zck)); @@ -1403,7 +1403,7 @@ cr_end_chunk(CR_FILE *cr_file, GError **err) #ifdef WITH_ZCHUNK zckCtx *zck = (zckCtx *) cr_file->FILE; ssize_t wb = zck_end_chunk(zck); - if(wb < 0) { + if (wb < 0) { g_set_error(err, ERR_DOMAIN, CRE_ZCK, "Error ending chunk: %s", zck_get_error(zck)); @@ -1454,7 +1454,7 @@ cr_set_autochunk(CR_FILE *cr_file, gboolean auto_chunk, GError **err) case (CR_CW_ZCK_COMPRESSION): { // ------------------------------------ #ifdef WITH_ZCHUNK zckCtx *zck = (zckCtx *) cr_file->FILE; - if(!zck_set_ioption(zck, ZCK_MANUAL_CHUNK, !auto_chunk)) { + if (!zck_set_ioption(zck, ZCK_MANUAL_CHUNK, !auto_chunk)) { g_set_error(err, ERR_DOMAIN, CRE_ZCK, "Error setting auto_chunk: %s", zck_get_error(zck)); diff --git a/src/createrepo_c.c b/src/createrepo_c.c index 5474aad7..92b0f4a1 100644 --- a/src/createrepo_c.c +++ b/src/createrepo_c.c @@ -837,28 +837,28 @@ main(int argc, char **argv) gchar *fil_dict_file = NULL; gchar *oth_dict_file = NULL; - if(cmd_options->zck_dict_dir) { + if (cmd_options->zck_dict_dir) { pri_dict_file = cr_get_dict_file(cmd_options->zck_dict_dir, "primary.xml"); fil_dict_file = cr_get_dict_file(cmd_options->zck_dict_dir, "filelists.xml"); oth_dict_file = cr_get_dict_file(cmd_options->zck_dict_dir, "other.xml"); - if(pri_dict_file && !g_file_get_contents(pri_dict_file, &pri_dict, + if (pri_dict_file && !g_file_get_contents(pri_dict_file, &pri_dict, &pri_dict_size, &tmp_err)) { g_critical("Error reading zchunk primary dict %s: %s", pri_dict_file, tmp_err->message); g_clear_error(&tmp_err); exit(EXIT_FAILURE); } - if(fil_dict_file && !g_file_get_contents(fil_dict_file, &fil_dict, + if (fil_dict_file && !g_file_get_contents(fil_dict_file, &fil_dict, &fil_dict_size, &tmp_err)) { g_critical("Error reading zchunk filelists dict %s: %s", fil_dict_file, tmp_err->message); g_clear_error(&tmp_err); exit(EXIT_FAILURE); } - if(oth_dict_file && !g_file_get_contents(oth_dict_file, &oth_dict, + if (oth_dict_file && !g_file_get_contents(oth_dict_file, &oth_dict, &oth_dict_size, &tmp_err)) { g_critical("Error reading zchunk other dict %s: %s", oth_dict_file, tmp_err->message); @@ -866,7 +866,7 @@ main(int argc, char **argv) exit(EXIT_FAILURE); } } - if(cmd_options->zck_compression) { + if (cmd_options->zck_compression) { g_debug("Creating .xml.zck files"); pri_zck_filename = g_strconcat(tmp_out_repo, "/primary.xml.zck", NULL); @@ -1013,7 +1013,7 @@ main(int argc, char **argv) g_thread_pool_free(pool, FALSE, TRUE); // if there were any errors, exit nonzero - if( cmd_options->error_exit_val && user_data.had_errors ) { + if ( cmd_options->error_exit_val && user_data.had_errors ) { exit_val = 2; } @@ -1253,7 +1253,7 @@ main(int argc, char **argv) } // Zchunk - if(cmd_options->zck_compression) { + if (cmd_options->zck_compression) { // Prepare repomd records pri_zck_rec = cr_repomd_record_new("primary_zck", pri_zck_filename); fil_zck_rec = cr_repomd_record_new("filelists_zck", fil_zck_filename); @@ -1318,7 +1318,7 @@ main(int argc, char **argv) exit(EXIT_FAILURE); } /* Only create updateinfo_zck if updateinfo isn't already zchunk */ - if(com_type != CR_CW_ZCK_COMPRESSION) { + if (com_type != CR_CW_ZCK_COMPRESSION) { updateinfo_zck_rec = cr_repomd_record_new("updateinfo_zck", updateinfo); cr_repomd_record_compress_and_fill(updateinfo_rec, updateinfo_zck_rec, @@ -1409,7 +1409,7 @@ main(int argc, char **argv) g_clear_error(&tmp_err); goto deltaerror; } - if(cmd_options->zck_compression && prestodelta_compression != CR_CW_ZCK_COMPRESSION) { + if (cmd_options->zck_compression && prestodelta_compression != CR_CW_ZCK_COMPRESSION) { filename = g_strconcat("prestodelta.xml", cr_compression_suffix(CR_CW_ZCK_COMPRESSION), NULL); @@ -1455,7 +1455,7 @@ main(int argc, char **argv) prestodelta_rec = cr_repomd_record_new("prestodelta", prestodelta_xml_filename); cr_repomd_record_load_contentstat(prestodelta_rec, prestodelta_stat); cr_repomd_record_fill(prestodelta_rec, cmd_options->repomd_checksum_type, NULL); - if(prestodelta_zck_stat) { + if (prestodelta_zck_stat) { prestodelta_zck_rec = cr_repomd_record_new("prestodelta_zck", prestodelta_zck_filename); cr_repomd_record_load_contentstat(prestodelta_zck_rec, prestodelta_zck_stat); cr_repomd_record_fill(prestodelta_zck_rec, cmd_options->repomd_checksum_type, NULL); diff --git a/src/deltarpms.c b/src/deltarpms.c index 53a9ea6f..96536a66 100644 --- a/src/deltarpms.c +++ b/src/deltarpms.c @@ -920,7 +920,7 @@ cr_deltarpms_generate_prestodelta_file(const gchar *drpmsdir, cr_xmlfile_add_chunk(f, chunk, NULL); /* Write out zchunk file */ - if(zck_f) { + if (zck_f) { cr_xmlfile_add_chunk(zck_f, chunk, NULL); cr_end_chunk(zck_f->f, &tmp_err); if (tmp_err) { diff --git a/src/dumper_thread.c b/src/dumper_thread.c index 2468c8e2..27c2493d 100644 --- a/src/dumper_thread.c +++ b/src/dumper_thread.c @@ -79,7 +79,7 @@ write_pkg(long id, udata->prev_srpm = udata->cur_srpm; udata->cur_srpm = g_strdup(pkg->rpm_sourcerpm); gboolean new_pkg = FALSE; - if(g_strcmp0(udata->prev_srpm, udata->cur_srpm) != 0) + if (g_strcmp0(udata->prev_srpm, udata->cur_srpm) != 0) new_pkg = TRUE; ++udata->id_pri; @@ -100,8 +100,8 @@ write_pkg(long id, g_clear_error(&tmp_err); } } - if(udata->pri_zck) { - if(new_pkg) { + if (udata->pri_zck) { + if (new_pkg) { cr_end_chunk(udata->pri_zck->f, &tmp_err); if (tmp_err) { g_critical("Unable to end primary zchunk: %s", tmp_err->message); @@ -144,7 +144,7 @@ write_pkg(long id, } } if (udata->fil_zck) { - if(new_pkg) { + if (new_pkg) { cr_end_chunk(udata->fil_zck->f, &tmp_err); if (tmp_err) { g_critical("Unable to end filelists zchunk: %s", tmp_err->message); @@ -187,7 +187,7 @@ write_pkg(long id, } } if (udata->oth_zck) { - if(new_pkg) { + if (new_pkg) { cr_end_chunk(udata->oth_zck->f, &tmp_err); if (tmp_err) { g_critical("Unable to end other zchunk: %s", tmp_err->message); diff --git a/src/locate_metadata.c b/src/locate_metadata.c index b4a04e6c..813a89cd 100644 --- a/src/locate_metadata.c +++ b/src/locate_metadata.c @@ -169,7 +169,7 @@ cr_get_remote_metadata(const char *repopath, gboolean ignore_sqlite) // Create temporary repo in /tmp tmp_dir = g_build_filename(g_get_tmp_dir(), TMPDIR_PATTERN, NULL); - if(!mkdtemp(tmp_dir)) { + if (!mkdtemp(tmp_dir)) { g_critical("%s: Cannot create a temporary directory: %s", __func__, g_strerror(errno)); return ret; diff --git a/src/mergerepo_c.c b/src/mergerepo_c.c index bd6dcee1..f971355c 100644 --- a/src/mergerepo_c.c +++ b/src/mergerepo_c.c @@ -425,7 +425,7 @@ check_arguments(struct CmdOptions *options) } // Zchunk options - if(options->zck_dict_dir && !options->zck_compression) { + if (options->zck_dict_dir && !options->zck_compression) { g_critical("Cannot use --zck-dict-dir without setting --zck"); ret = FALSE; } @@ -1173,28 +1173,28 @@ dump_merged_metadata(GHashTable *merged_hashtable, gchar *fil_dict_file = NULL; gchar *oth_dict_file = NULL; - if(cmd_options->zck_dict_dir) { + if (cmd_options->zck_dict_dir) { pri_dict_file = cr_get_dict_file(cmd_options->zck_dict_dir, "primary.xml"); fil_dict_file = cr_get_dict_file(cmd_options->zck_dict_dir, "filelists.xml"); oth_dict_file = cr_get_dict_file(cmd_options->zck_dict_dir, "other.xml"); - if(pri_dict_file && !g_file_get_contents(pri_dict_file, &pri_dict, + if (pri_dict_file && !g_file_get_contents(pri_dict_file, &pri_dict, &pri_dict_size, &tmp_err)) { g_critical("Error reading zchunk primary dict %s: %s", pri_dict_file, tmp_err->message); g_clear_error(&tmp_err); exit(EXIT_FAILURE); } - if(fil_dict_file && !g_file_get_contents(fil_dict_file, &fil_dict, + if (fil_dict_file && !g_file_get_contents(fil_dict_file, &fil_dict, &fil_dict_size, &tmp_err)) { g_critical("Error reading zchunk filelists dict %s: %s", fil_dict_file, tmp_err->message); g_clear_error(&tmp_err); exit(EXIT_FAILURE); } - if(oth_dict_file && !g_file_get_contents(oth_dict_file, &oth_dict, + if (oth_dict_file && !g_file_get_contents(oth_dict_file, &oth_dict, &oth_dict_size, &tmp_err)) { g_critical("Error reading zchunk other dict %s: %s", oth_dict_file, tmp_err->message); @@ -1278,7 +1278,7 @@ dump_merged_metadata(GHashTable *merged_hashtable, cr_xmlfile_set_num_of_pkgs(fil_f, packages, NULL); cr_xmlfile_set_num_of_pkgs(oth_f, packages, NULL); - if(cmd_options->zck_compression) { + if (cmd_options->zck_compression) { g_debug("Creating .xml.zck files"); pri_zck_filename = g_strconcat(cmd_options->tmp_out_repo, @@ -1423,7 +1423,7 @@ dump_merged_metadata(GHashTable *merged_hashtable, g_debug("Writing metadata for %s (%s-%s.%s)", pkg->name, pkg->version, pkg->release, pkg->arch); - if(cmd_options->zck_compression && + if (cmd_options->zck_compression && (!prev_srpm || !pkg->rpm_sourcerpm || strlen(prev_srpm) != strlen(pkg->rpm_sourcerpm) || strncmp(pkg->rpm_sourcerpm, prev_srpm, strlen(prev_srpm)) != 0)) { @@ -1431,7 +1431,7 @@ dump_merged_metadata(GHashTable *merged_hashtable, cr_end_chunk(fil_cr_zck->f, NULL); cr_end_chunk(oth_cr_zck->f, NULL); g_free(prev_srpm); - if(pkg->rpm_sourcerpm) + if (pkg->rpm_sourcerpm) prev_srpm = g_strdup(pkg->rpm_sourcerpm); else prev_srpm = NULL; @@ -1439,7 +1439,7 @@ dump_merged_metadata(GHashTable *merged_hashtable, cr_xmlfile_add_chunk(pri_f, (const char *) res.primary, NULL); cr_xmlfile_add_chunk(fil_f, (const char *) res.filelists, NULL); cr_xmlfile_add_chunk(oth_f, (const char *) res.other, NULL); - if(cmd_options->zck_compression) { + if (cmd_options->zck_compression) { cr_xmlfile_add_chunk(pri_cr_zck, (const char *) res.primary, NULL); cr_xmlfile_add_chunk(fil_cr_zck, (const char *) res.filelists, NULL); cr_xmlfile_add_chunk(oth_cr_zck, (const char *) res.other, NULL); @@ -1465,7 +1465,7 @@ dump_merged_metadata(GHashTable *merged_hashtable, cr_xmlfile_close(pri_f, NULL); cr_xmlfile_close(fil_f, NULL); cr_xmlfile_close(oth_f, NULL); - if(cmd_options->zck_compression) { + if (cmd_options->zck_compression) { cr_xmlfile_close(pri_cr_zck, NULL); cr_xmlfile_close(fil_cr_zck, NULL); cr_xmlfile_close(oth_cr_zck, NULL); @@ -1558,7 +1558,7 @@ dump_merged_metadata(GHashTable *merged_hashtable, CR_CHECKSUM_SHA256, cmd_options->groupfile_compression_type, NULL, NULL); - if(cmd_options->zck_compression) { + if (cmd_options->zck_compression) { groupfile_zck_rec = cr_repomd_record_new("group_zck", groupfile); cr_repomd_record_compress_and_fill(groupfile_rec, groupfile_zck_rec, @@ -1574,7 +1574,7 @@ dump_merged_metadata(GHashTable *merged_hashtable, if (!cmd_options->noupdateinfo) { update_info_rec = cr_repomd_record_new("updateinfo", update_info_filename); cr_repomd_record_fill(update_info_rec, CR_CHECKSUM_SHA256, NULL); - if(cmd_options->zck_compression) { + if (cmd_options->zck_compression) { update_info_zck_rec = cr_repomd_record_new("updateinfo_zck", update_info_filename); cr_repomd_record_compress_and_fill(update_info_rec, update_info_zck_rec, @@ -1591,7 +1591,7 @@ dump_merged_metadata(GHashTable *merged_hashtable, gchar *pkgorigins_path = g_strconcat(cmd_options->tmp_out_repo, "pkgorigins.gz", NULL); pkgorigins_rec = cr_repomd_record_new("origin", pkgorigins_path); cr_repomd_record_fill(pkgorigins_rec, CR_CHECKSUM_SHA256, NULL); - if(cmd_options->zck_compression) { + if (cmd_options->zck_compression) { pkgorigins_zck_rec = cr_repomd_record_new("origin_zck", pkgorigins_path); cr_repomd_record_compress_and_fill(pkgorigins_rec, pkgorigins_zck_rec, @@ -1719,7 +1719,7 @@ dump_merged_metadata(GHashTable *merged_hashtable, } // Zchunk - if(cmd_options->zck_compression) { + if (cmd_options->zck_compression) { // Prepare repomd records pri_zck_rec = cr_repomd_record_new("primary_zck", pri_zck_filename); fil_zck_rec = cr_repomd_record_new("filelists_zck", fil_zck_filename); diff --git a/src/misc.c b/src/misc.c index 710782a4..47c4c9f4 100644 --- a/src/misc.c +++ b/src/misc.c @@ -470,10 +470,10 @@ cr_compress_file_with_stat(const char *src, NULL); } else if (c_suffix && !g_str_has_suffix(dst, c_suffix)) { cr_CompressionType old_type = cr_detect_compression(src, &tmp_err); - if(tmp_err) { + if (tmp_err) { g_debug("%s: Unable to detect compression type of %s", __func__, src); g_clear_error(&tmp_err); - } else if(old_type != CR_CW_NO_COMPRESSION) { + } else if (old_type != CR_CW_NO_COMPRESSION) { _cleanup_free_ gchar *tmp_file = g_strndup(dst, strlen(dst) - strlen(cr_compression_suffix(old_type))); *in_dst = g_strconcat(tmp_file, c_suffix, @@ -485,7 +485,7 @@ cr_compress_file_with_stat(const char *src, dst = (gchar *) *in_dst; int mode = CR_CW_NO_COMPRESSION; - if(compression == CR_CW_ZCK_COMPRESSION) + if (compression == CR_CW_ZCK_COMPRESSION) mode = CR_CW_AUTO_DETECT_COMPRESSION; orig = cr_open(src, @@ -500,12 +500,12 @@ cr_compress_file_with_stat(const char *src, _cleanup_free_ gchar *dict = NULL; size_t dict_size = 0; - if(compression == CR_CW_ZCK_COMPRESSION && zck_dict_dir) { + if (compression == CR_CW_ZCK_COMPRESSION && zck_dict_dir) { /* Find zdict */ _cleanup_free_ gchar *file_basename = NULL; - if(dst) { + if (dst) { _cleanup_free_ gchar *dict_base = NULL; - if(g_str_has_suffix(dst, ".zck")) + if (g_str_has_suffix(dst, ".zck")) dict_base = g_strndup(dst, strlen(dst)-4); else dict_base = g_strdup(dst); @@ -516,7 +516,7 @@ cr_compress_file_with_stat(const char *src, _cleanup_free_ gchar *dict_file = cr_get_dict_file(zck_dict_dir, file_basename); /* Read dictionary from file */ - if(dict_file && !g_file_get_contents(dict_file, &dict, + if (dict_file && !g_file_get_contents(dict_file, &dict, &dict_size, &tmp_err)) { g_set_error(err, ERR_DOMAIN, CRE_IO, "Error reading zchunk dict %s: %s", @@ -533,13 +533,13 @@ cr_compress_file_with_stat(const char *src, ret = CRE_IO; goto compress_file_cleanup; } - if(compression == CR_CW_ZCK_COMPRESSION) { - if(dict && cr_set_dict(new, dict, dict_size, &tmp_err) != CRE_OK) { + if (compression == CR_CW_ZCK_COMPRESSION) { + if (dict && cr_set_dict(new, dict, dict_size, &tmp_err) != CRE_OK) { ret = tmp_err->code; g_propagate_prefixed_error(err, tmp_err, "Unable to set zdict for %s: ", dst); goto compress_file_cleanup; } - if(zck_auto_chunk && cr_set_autochunk(new, TRUE, &tmp_err) != CRE_OK) { + if (zck_auto_chunk && cr_set_autochunk(new, TRUE, &tmp_err) != CRE_OK) { ret = tmp_err->code; g_propagate_prefixed_error(err, tmp_err, "Unable to set auto-chunking for %s: ", dst); goto compress_file_cleanup; @@ -1570,7 +1570,7 @@ cr_get_dict_file(const gchar *dir, const gchar *file) assert(full_path); free(dict_file); - if(!g_file_test(full_path, G_FILE_TEST_EXISTS)) { + if (!g_file_test(full_path, G_FILE_TEST_EXISTS)) { g_warning("%s: Zchunk dict %s doesn't exist", __func__, full_path); free(full_path); return NULL; diff --git a/src/modifyrepo_c.c b/src/modifyrepo_c.c index dc8954fe..c127c3de 100644 --- a/src/modifyrepo_c.c +++ b/src/modifyrepo_c.c @@ -176,7 +176,7 @@ check_arguments(RawCmdOptions *options, GError **err) } // Zchunk options - if(options->zck_dict_dir && !options->zck) { + if (options->zck_dict_dir && !options->zck) { g_set_error(err, ERR_DOMAIN, CRE_ERROR, "Cannot use --zck-dict-dir without setting --zck"); return FALSE; diff --git a/src/modifyrepo_shared.c b/src/modifyrepo_shared.c index 93a8965d..73fb3a5e 100644 --- a/src/modifyrepo_shared.c +++ b/src/modifyrepo_shared.c @@ -248,7 +248,7 @@ cr_modifyrepo(GSList *modifyrepotasks, gchar *repopath, GError **err) compress_type = task->compress_type; dst_fn = write_file(repopath, task, compress_type, err); - if(dst_fn == NULL) { + if (dst_fn == NULL) { cr_repomd_free(repomd); g_free(repomd_path); return FALSE; @@ -256,10 +256,10 @@ cr_modifyrepo(GSList *modifyrepotasks, gchar *repopath, GError **err) task->repopath = cr_safe_string_chunk_insert_null(task->chunk, dst_fn); #ifdef WITH_ZCHUNK - if(task->zck) { + if (task->zck) { free(dst_fn); dst_fn = write_file(repopath, task, CR_CW_ZCK_COMPRESSION, err); - if(dst_fn == NULL) { + if (dst_fn == NULL) { cr_repomd_free(repomd); g_free(repomd_path); return FALSE; @@ -295,7 +295,7 @@ cr_modifyrepo(GSList *modifyrepotasks, gchar *repopath, GError **err) repomdrecords_uniquefn = g_slist_prepend(repomdrecords_uniquefn, rec); repomdrecordfilltasks = g_slist_prepend(repomdrecordfilltasks, filltask); - if(task->zck) { + if (task->zck) { _cleanup_free_ gchar *type = g_strconcat(task->type, "_zck", NULL); rec = cr_repomd_record_new(type, task->zck_repopath); filltask = cr_repomdrecordfilltask_new(rec, task->checksum_type, NULL); @@ -332,7 +332,7 @@ cr_modifyrepo(GSList *modifyrepotasks, gchar *repopath, GError **err) __func__, task->type); recordstoremove = g_slist_prepend(recordstoremove, rec); cr_repomd_detach_record(repomd, rec); - if(task->zck) { + if (task->zck) { _cleanup_free_ gchar *type = g_strconcat(task->type, "_zck", NULL); cr_RepomdRecord *rec = cr_repomd_get_record(repomd, type); if (rec) { diff --git a/src/repomd.c b/src/repomd.c index cb11696d..2d4a6f34 100644 --- a/src/repomd.c +++ b/src/repomd.c @@ -110,10 +110,10 @@ cr_repomd_record_copy(const cr_RepomdRecord *orig) rec->size_open = orig->size_open; rec->size_header = orig->size_header; rec->db_ver = orig->db_ver; - if(orig->checksum_header) + if (orig->checksum_header) rec->checksum_header = cr_safe_string_chunk_insert(rec->chunk, orig->checksum_header); - if(orig->checksum_header_type) + if (orig->checksum_header_type) rec->checksum_header_type = cr_safe_string_chunk_insert(rec->chunk, orig->checksum_header_type); @@ -185,7 +185,7 @@ cr_get_compressed_content_stat(const char *filename, contentStat* result = g_malloc0(sizeof(contentStat)); if (result) { - if(cwfile->stat) { + if (cwfile->stat) { result->hdr_checksum = cwfile->stat->hdr_checksum; result->hdr_checksum_type = cwfile->stat->hdr_checksum_type; result->hdr_size = cwfile->stat->hdr_size; @@ -286,7 +286,7 @@ cr_repomd_record_fill(cr_RepomdRecord *md, md->checksum_open = g_string_chunk_insert(md->chunk, open_stat->checksum); if (md->size_open == G_GINT64_CONSTANT(-1)) md->size_open = open_stat->size; - if(open_stat->hdr_checksum != NULL) { + if (open_stat->hdr_checksum != NULL) { const char *hdr_checksum_str = cr_checksum_name_str(open_stat->hdr_checksum_type); md->checksum_header_type = g_string_chunk_insert(md->chunk, hdr_checksum_str); @@ -402,7 +402,7 @@ cr_repomd_record_compress_and_fill(cr_RepomdRecord *record, // Compress file + get size of non compressed file int mode = CR_CW_NO_COMPRESSION; - if(record_compression == CR_CW_ZCK_COMPRESSION) + if (record_compression == CR_CW_ZCK_COMPRESSION) mode = CR_CW_AUTO_DETECT_COMPRESSION; cw_plain = cr_open(path, @@ -417,18 +417,18 @@ cr_repomd_record_compress_and_fill(cr_RepomdRecord *record, _cleanup_free_ gchar *dict = NULL; size_t dict_size = 0; - if(record_compression == CR_CW_ZCK_COMPRESSION && zck_dict_dir) { + if (record_compression == CR_CW_ZCK_COMPRESSION && zck_dict_dir) { /* Find zdict */ _cleanup_free_ gchar *file_basename = NULL; _cleanup_free_ gchar *dict_base = NULL; - if(g_str_has_suffix(cpath, ".zck")) + if (g_str_has_suffix(cpath, ".zck")) dict_base = g_strndup(cpath, strlen(cpath)-4); else dict_base = g_strdup(cpath); file_basename = g_path_get_basename(dict_base); _cleanup_free_ gchar *dict_file = cr_get_dict_file(zck_dict_dir, file_basename); /* Read dictionary from file */ - if(dict_file && !g_file_get_contents(dict_file, &dict, + if (dict_file && !g_file_get_contents(dict_file, &dict, &dict_size, &tmp_err)) { ret = tmp_err->code; g_propagate_prefixed_error(err, tmp_err, "Error reading zchunk dict %s:", dict_file); @@ -448,13 +448,13 @@ cr_repomd_record_compress_and_fill(cr_RepomdRecord *record, return ret; } - if(record_compression == CR_CW_ZCK_COMPRESSION) { - if(dict && cr_set_dict(cw_compressed, dict, dict_size, &tmp_err) != CRE_OK) { + if (record_compression == CR_CW_ZCK_COMPRESSION) { + if (dict && cr_set_dict(cw_compressed, dict, dict_size, &tmp_err) != CRE_OK) { ret = tmp_err->code; g_propagate_prefixed_error(err, tmp_err, "Unable to set zdict for %s: ", cpath); return ret; } - if(cr_set_autochunk(cw_compressed, TRUE, &tmp_err) != CRE_OK) { + if (cr_set_autochunk(cw_compressed, TRUE, &tmp_err) != CRE_OK) { ret = tmp_err->code; g_propagate_prefixed_error(err, tmp_err, "Unable to set auto-chunking for %s: ", cpath); return ret; @@ -529,7 +529,7 @@ cr_repomd_record_compress_and_fill(cr_RepomdRecord *record, cgf_size = cgf_stat.st_size; cgf_time = cgf_stat.st_mtime; - if(out_stat->hdr_checksum) { + if (out_stat->hdr_checksum) { cgf_hdrsize = out_stat->hdr_size; hdr_checksum_str = cr_checksum_name_str(out_stat->hdr_checksum_type); hdrchecksum = out_stat->hdr_checksum; @@ -552,7 +552,7 @@ cr_repomd_record_compress_and_fill(cr_RepomdRecord *record, crecord->checksum_type = g_string_chunk_insert(crecord->chunk, checksum_str); crecord->checksum_open = g_string_chunk_insert(record->chunk, checksum); crecord->checksum_open_type = g_string_chunk_insert(record->chunk, checksum_str); - if(hdr_checksum_str) { + if (hdr_checksum_str) { crecord->checksum_header = g_string_chunk_insert(crecord->chunk, hdrchecksum); crecord->checksum_header_type = g_string_chunk_insert(crecord->chunk, hdr_checksum_str); } else { @@ -798,7 +798,7 @@ cr_repomd_set_record(cr_Repomd *repomd, cr_RepomdRecord *delrec = NULL; // Remove all existing record of the same type - while((delrec = cr_repomd_get_record(repomd, record->type)) != NULL) { + while ((delrec = cr_repomd_get_record(repomd, record->type)) != NULL) { cr_repomd_detach_record(repomd, delrec); cr_repomd_record_free(delrec); } diff --git a/src/threads.c b/src/threads.c index 54f8416f..30f85909 100644 --- a/src/threads.c +++ b/src/threads.c @@ -60,7 +60,7 @@ cr_compressiontask_new(const char *src, task->dst = g_strdup(dst); task->type = compression_type; task->stat = stat; - if(zck_dict_dir != NULL) + if (zck_dict_dir != NULL) task->zck_dict_dir = g_strdup(zck_dict_dir); task->zck_auto_chunk = zck_auto_chunk; task->delsrc = delsrc;