Skip to content

Commit

Permalink
core: implement a function computing chunk IDs from oio_url_s
Browse files Browse the repository at this point in the history
  • Loading branch information
@fvennetier
fvennetier committed Mar 26, 2021
1 parent 9b37987 commit 5ac57dd
Show file tree
Hide file tree
Showing 3 changed files with 187 additions and 11 deletions.
66 changes: 55 additions & 11 deletions core/url.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
OpenIO SDS core library
Copyright (C) 2014 Worldline, as part of Redcurrant
Copyright (C) 2015-2018 OpenIO SAS, as part of OpenIO SDS
Copyright (C) 2021 OVH SAS
This library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
Expand All @@ -23,6 +24,7 @@ License along with this library.

#include <core/oiostr.h>
#include <core/oioext.h>
#include <core/internals.h>
#include <core/url_ext.h>
#include <core/client_variables.h>
#include <core/oiolog.h>
Expand Down Expand Up @@ -112,22 +114,22 @@ _check_parsed_url (struct oio_url_s *u)
static void
_replace (gchar **pp, const char *s, const gboolean unescape)
{
if (unescape)
oio_str_reuse(pp, g_uri_unescape_string (s, NULL));
else
oio_str_reuse(pp, g_strdup(s));
if (unescape)
oio_str_reuse(pp, g_uri_unescape_string (s, NULL));
else
oio_str_reuse(pp, g_strdup(s));
}

static void
_copy(gchar *dst, gsize dstlen, const char *src, const gboolean unescape)
{
if (!unescape) {
g_strlcpy(dst, src, dstlen);
} else {
gchar *tmp = g_uri_unescape_string (src, NULL);
g_strlcpy(dst, tmp, dstlen);
g_free(tmp);
}
if (!unescape) {
g_strlcpy(dst, src, dstlen);
} else {
gchar *tmp = g_uri_unescape_string (src, NULL);
g_strlcpy(dst, tmp, dstlen);
g_free(tmp);
}
}

static int
Expand Down Expand Up @@ -678,3 +680,45 @@ oio_url_check(const struct oio_url_s *u, const char *namespace,

#undef _ERR
}

GError *
oio_url_compute_chunk_id(struct oio_url_s *url, const char *position,
const char *policy, char *out, size_t outsize)
{
g_assert_nonnull(out);

const gchar *hexid = oio_url_get(url, OIOURL_HEXID);
const gchar *alias = oio_url_get(url, OIOURL_PATH);
const gchar *version = oio_url_get(url, OIOURL_VERSION);

if (!oio_str_is_set(hexid)) {
return BADREQ("Null or empty container ID");
}
if (!oio_str_is_set(alias)) {
return BADREQ("Null or empty object alias");
}
if (!oio_str_is_set(version)) {
return BADREQ("Missing object version");
}
if (!oio_str_is_set(position)) {
return BADREQ("Missing chunk position");
}
if (!oio_str_is_set(policy)) {
return BADREQ("Missing object storage policy");
}

GChecksum *sum = g_checksum_new(G_CHECKSUM_SHA256);
g_checksum_update(sum, (guint8*)hexid, -1);
g_checksum_update(sum, (guint8*)alias, -1);
g_checksum_update(sum, (guint8*)version, -1);
g_checksum_update(sum, (guint8*)position, -1);
g_checksum_update(sum, (guint8*)policy, -1);

gsize sz = 64;
guint8 buf[sz];
g_checksum_get_digest(sum, buf, &sz);
g_checksum_free(sum);
oio_str_bin2hex(buf, sz, out, outsize);

return NULL;
}
14 changes: 14 additions & 0 deletions core/url_ext.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,4 +48,18 @@ gboolean oio_requri_parse (const char *packed, struct oio_requri_s *ruri);

void oio_requri_clear (struct oio_requri_s *ruri);

/**
* Compute the ID of the chunk at the specified position for the specified
* storage policy.
*
* @param u the URL
* @param position the position of the chunk, simple ("1") or composed ("1.1")
* @param policy the name of the storage policy
* @param out an output buffer
* @param outsize size of the output buffer. In case the ID is longer than the
* buffer, it will be truncated.
*/
GError *oio_url_compute_chunk_id(struct oio_url_s *u, const char *position,
const char *policy, char *out, size_t outsize);

#endif /*OIO_SDS__metautils__lib__oio_url_ext_h*/
118 changes: 118 additions & 0 deletions tests/unit/test_url.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,12 @@ License along with this library.
#include <stdio.h>

#include <core/oio_core.h>
#include <core/url_ext.h>
#include <metautils/lib/common_main.h>
#include <core/internals.h>

#undef GQ
#define GQ() g_quark_from_static_string("oio.core")

struct test_data_s {
const char *url; /* to be decoded */
Expand Down Expand Up @@ -217,13 +222,126 @@ test_dup (void)
oio_url_pclean (&u0);
}

/* Test the generation of chunk IDs depending on object name */
static void
test_chunk_id_generation(void)
{
char buf[65] = {0};
struct oio_url_s *url = oio_url_init("NS/ACCT/FVE/1/obj");
const char expected_id[] =
"9EA177B73CDE093ED3530DA1A06BA7DA1F038327FA5F2A56B5417140B8A616BD";
const char expected_id_1[] =
"A69598A38612D0B126EA0BD55A538362D454BAE6555F8348251A1C3D1F5EDF42";
const char expected_id_2[] =
"F67A7270D181449C6DDC547A4233AF29B6C9C2055DCBF9326322AB8C118C7942";

g_assert_no_error(
oio_url_compute_chunk_id(url, "0", "SINGLE", buf, sizeof(buf)));
g_assert_cmpint(64, ==, strlen(buf));
g_assert_cmpstr(expected_id, ==, buf);

/* Short buffer */
g_assert_no_error(
oio_url_compute_chunk_id(url, "0", "SINGLE", buf, 33));
g_assert_cmpint(32, ==, strlen(buf));
g_assert_cmpstr("9EA177B73CDE093ED3530DA1A06BA7DA", ==, buf);

/* Different position */
g_assert_no_error(
oio_url_compute_chunk_id(url, "1", "SINGLE", buf, 33));
g_assert_cmpint(32, ==, strlen(buf));
g_assert_cmpstr("9EA177B73CDE093ED3530DA1A06BA7DA", !=, buf);
g_assert_no_error(
oio_url_compute_chunk_id(url, "0.1", "SINGLE", buf, 33));
g_assert_cmpint(32, ==, strlen(buf));
g_assert_cmpstr("9EA177B73CDE093ED3530DA1A06BA7DA", !=, buf);
g_assert_no_error(
oio_url_compute_chunk_id(url, "11.11", "SINGLE", buf, 33));
g_assert_cmpint(32, ==, strlen(buf));
g_assert_cmpstr("9EA177B73CDE093ED3530DA1A06BA7DA", !=, buf);

/* Different policy */
g_assert_no_error(
oio_url_compute_chunk_id(url, "0", "DUP", buf, 33));
g_assert_cmpint(32, ==, strlen(buf));
g_assert_cmpstr("9EA177B73CDE093ED3530DA1A06BA7DA", !=, buf);
g_assert_no_error(
oio_url_compute_chunk_id(url, "0", "THREECOPIES", buf, 33));
g_assert_cmpint(32, ==, strlen(buf));
g_assert_cmpstr("9EA177B73CDE093ED3530DA1A06BA7DA", !=, buf);

/* Different version */
oio_url_set(url, OIOURL_VERSION, "2");
g_assert_no_error(
oio_url_compute_chunk_id(url, "0", "SINGLE", buf, sizeof(buf)));
g_assert_cmpint(64, ==, strlen(buf));
g_assert_cmpstr(expected_id_1, ==, buf);

oio_url_set(url, OIOURL_VERSION, "3");
g_assert_no_error(
oio_url_compute_chunk_id(url, "0", "SINGLE", buf, sizeof(buf)));
g_assert_cmpint(64, ==, strlen(buf));
g_assert_cmpstr(expected_id_2, ==, buf);

oio_url_pclean (&url);
}

static void
test_chunk_id_generation_invalid_input(void)
{
char buf[33] = {0};
struct oio_url_s *url = NULL;
GError *err = NULL;

/* URL without object name */
url = oio_url_init("NS/ACCT/FVE");
err = oio_url_compute_chunk_id(url, "0", "SINGLE", buf, sizeof(buf));
g_assert_error(err, GQ(), CODE_BAD_REQUEST);
g_assert_nonnull(g_strstr_len(err->message, -1, "object alias"));
g_clear_error(&err);
oio_url_pclean(&url);

/* No version */
url = oio_url_init("NS/ACCT/FVE//obj");
err = oio_url_compute_chunk_id(url, NULL, "SINGLE", buf, sizeof(buf));
g_assert_error(err, GQ(), CODE_BAD_REQUEST);
g_assert_nonnull(g_strstr_len(err->message, -1, "object version"));
g_clear_error(&err);
oio_url_pclean(&url);

/* No position */
url = oio_url_init("NS/ACCT/FVE/1/obj");
err = oio_url_compute_chunk_id(url, NULL, "SINGLE", buf, sizeof(buf));
g_assert_error(err, GQ(), CODE_BAD_REQUEST);
g_assert_nonnull(g_strstr_len(err->message, -1, "chunk position"));
g_clear_error(&err);

/* No storage policy */
err = oio_url_compute_chunk_id(url, "0", NULL, buf, sizeof(buf));
g_assert_error(err, GQ(), CODE_BAD_REQUEST);
g_assert_nonnull(g_strstr_len(err->message, -1, "storage policy"));
g_clear_error(&err);
oio_url_pclean(&url);

/* Empty URL */
url = oio_url_empty();
err = oio_url_compute_chunk_id(url, "0", "SINGLE", buf, sizeof(buf));
g_assert_error(err, GQ(), CODE_BAD_REQUEST);
g_assert_nonnull(g_strstr_len(err->message, -1, "container ID"));
g_clear_error(&err);
oio_url_pclean(&url);
}

int
main(int argc, char **argv)
{
HC_TEST_INIT(argc,argv);
g_test_add_func("/core/url/configure/valid", test_configure_valid);
g_test_add_func("/core/url/configure/invalid", test_configure_invalid);
g_test_add_func("/core/url/dup", test_dup);
g_test_add_func("/core/url/chunkid/valid", test_chunk_id_generation);
g_test_add_func("/core/url/chunkid/invalid",
test_chunk_id_generation_invalid_input);
return g_test_run();
}

0 comments on commit 5ac57dd

Please sign in to comment.