Skip to content
Permalink
Browse files

Rewrite xz plugin as a filter.

The xz plugin should really be a filter so that it can be applied on
top of other sources, especially curl:

  nbdkit --filter=xz curl url=https://download.fedoraproject.org/pub/fedora/linux/releases/29/Cloud/x86_64/images/Fedora-Cloud-Base-29-1.2.x86_64.raw.xz

This repurposes the xz test to test only the filter.  The plugin is no
longer tested as it will be deprecated in the next commit.
  • Loading branch information...
rwmjones committed Nov 21, 2018
1 parent b03837a commit c879d31047ae149cedabd5abbdb0a95e610f01c5
Showing with 1,189 additions and 26 deletions.
  1. +0 −2 TODO
  2. +5 −3 configure.ac
  3. +71 −0 filters/xz/Makefile.am
  4. +146 −0 filters/xz/blkcache.c
  5. +50 −0 filters/xz/blkcache.h
  6. +93 −0 filters/xz/nbdkit-xz-filter.pod
  7. +230 −0 filters/xz/xz.c
  8. +502 −0 filters/xz/xzfile.c
  9. +71 −0 filters/xz/xzfile.h
  10. +19 −19 tests/Makefile.am
  11. +2 −2 tests/test-xz.c
2 TODO
@@ -90,8 +90,6 @@ Suggestions for filters

* tar plugin should really be a filter

* xz plugin should really be a filter

* gzip plugin should really be a filter

* masking plugin features for testing clients (see 'nozero' and 'fua'
@@ -599,10 +599,10 @@ AS_IF([test "$with_zlib" != "no"],[
])
AM_CONDITIONAL([HAVE_ZLIB],[test "x$ZLIB_LIBS" != "x"])

dnl Check for liblzma (only if you want to compile the xz plugin).
dnl Check for liblzma (only if you want to compile the xz filter).
AC_ARG_WITH([liblzma],
[AS_HELP_STRING([--without-liblzma],
[disable xz plugin @<:@default=check@:>@])],
[disable xz filter @<:@default=check@:>@])],
[],
[with_liblzma=check])
AS_IF([test "$with_liblzma" != "no"],[
@@ -611,7 +611,7 @@ AS_IF([test "$with_liblzma" != "no"],[
AC_SUBST([LIBLZMA_LIBS])
AC_DEFINE([HAVE_LIBLZMA],[1],[liblzma found at compile time.])
],
[AC_MSG_WARN([liblzma not found, xz plugin will be disabled])])
[AC_MSG_WARN([liblzma not found, xz filter will be disabled])])
])
AM_CONDITIONAL([HAVE_LIBLZMA],[test "x$LIBLZMA_LIBS" != "x"])

@@ -728,6 +728,7 @@ filters="\
offset \
partition \
truncate \
xz \
"
AC_SUBST([plugins])
AC_SUBST([lang_plugins])
@@ -790,6 +791,7 @@ AC_CONFIG_FILES([Makefile
filters/offset/Makefile
filters/partition/Makefile
filters/truncate/Makefile
filters/xz/Makefile
src/Makefile
src/nbdkit.pc
tests/functions.sh
@@ -0,0 +1,71 @@
# nbdkit
# Copyright (C) 2013-2018 Red Hat Inc.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
# * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
#
# * Redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in the
# documentation and/or other materials provided with the distribution.
#
# * Neither the name of Red Hat nor the names of its contributors may be
# used to endorse or promote products derived from this software without
# specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY RED HAT AND CONTRIBUTORS ''AS IS'' AND
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
# PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL RED HAT OR
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
# SUCH DAMAGE.

include $(top_srcdir)/common-rules.mk

EXTRA_DIST = nbdkit-xz-filter.pod

if HAVE_LIBLZMA

filter_LTLIBRARIES = nbdkit-xz-filter.la

nbdkit_xz_filter_la_SOURCES = \
blkcache.c \
blkcache.h \
xz.c \
xzfile.c \
xzfile.h \
$(top_srcdir)/include/nbdkit-filter.h

nbdkit_xz_filter_la_CPPFLAGS = \
-I$(top_srcdir)/include
nbdkit_xz_filter_la_CFLAGS = \
$(WARNINGS_CFLAGS) \
$(LIBLZMA_CFLAGS)
nbdkit_xz_filter_la_LIBADD = \
$(LIBLZMA_LIBS)
nbdkit_xz_filter_la_LDFLAGS = \
-module -avoid-version -shared

if HAVE_POD

man_MANS = nbdkit-xz-filter.1
CLEANFILES += $(man_MANS)

nbdkit-xz-filter.1: nbdkit-xz-filter.pod
$(PODWRAPPER) --section=1 --man $@ \
--html $(top_builddir)/html/$@.html \
$<

endif HAVE_POD

endif
@@ -0,0 +1,146 @@
/* nbdkit
* Copyright (C) 2013 Red Hat Inc.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are
* met:
*
* * Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
*
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* * Neither the name of Red Hat nor the names of its contributors may be
* used to endorse or promote products derived from this software without
* specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY RED HAT AND CONTRIBUTORS ''AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
* THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
* PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL RED HAT OR
* CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
* USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
* ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/

#include <config.h>

#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <stdint.h>
#include <inttypes.h>

#include <nbdkit-plugin.h>

#include "blkcache.h"

/* Implemented as a very simple LRU list with a fixed depth. */
struct blkcache {
size_t maxdepth;
struct block *blocks;
blkcache_stats stats;
};

struct block {
uint64_t start;
uint64_t size;
char *data;
};

blkcache *
new_blkcache (size_t maxdepth)
{
blkcache *c;

c = malloc (sizeof *c);
if (!c) {
nbdkit_error ("malloc: %m");
return NULL;
}

c->blocks = calloc (maxdepth, sizeof (struct block));
if (!c->blocks) {
nbdkit_error ("calloc: %m");
free (c);
return NULL;
}
c->maxdepth = maxdepth;
c->stats.hits = c->stats.misses = 0;

return c;
}

void
free_blkcache (blkcache *c)
{
size_t i;

for (i = 0; i < c->maxdepth; ++i)
free (c->blocks[i].data);
free (c->blocks);
free (c);
}

char *
get_block (blkcache *c, uint64_t offset, uint64_t *start, uint64_t *size)
{
size_t i;
struct block tmp;

for (i = 0; i < c->maxdepth; ++i) {
if (c->blocks[i].data != NULL &&
c->blocks[i].start <= offset &&
offset < c->blocks[i].start + c->blocks[i].size) {
/* This block is now most recently used, so put it at the start. */
if (i > 0) {
tmp = c->blocks[0];
c->blocks[0] = c->blocks[i];
c->blocks[i] = tmp;
}

c->stats.hits++;
*start = c->blocks[0].start;
*size = c->blocks[0].size;
return c->blocks[0].data;
}
}

c->stats.misses++;
return NULL;
}

int
put_block (blkcache *c, uint64_t start, uint64_t size, char *data)
{
size_t i;

/* Eject the least recently used block. */
i = c->maxdepth-1;
if (c->blocks[i].data != NULL)
free (c->blocks[i].data);

for (; i >= 1; --i)
c->blocks[i] = c->blocks[i-1];

/* The new block is most recently used, so it goes at the start. */
c->blocks[0].start = start;
c->blocks[0].size = size;
c->blocks[0].data = data;

return 0;
}

void
blkcache_get_stats (blkcache *c, blkcache_stats *ret)
{
memcpy (ret, &c->stats, sizeof (c->stats));
}
@@ -0,0 +1,50 @@
/* nbdkit
* Copyright (C) 2013 Red Hat Inc.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are
* met:
*
* * Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
*
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* * Neither the name of Red Hat nor the names of its contributors may be
* used to endorse or promote products derived from this software without
* specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY RED HAT AND CONTRIBUTORS ''AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
* THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
* PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL RED HAT OR
* CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
* USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
* ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/

#ifndef NBDKIT_BLKCACHE_H
#define NBDKIT_BLKCACHE_H

typedef struct blkcache blkcache;

typedef struct blkcache_stats {
size_t hits;
size_t misses;
} blkcache_stats;

extern blkcache *new_blkcache (size_t maxdepth);
extern void free_blkcache (blkcache *);
extern char *get_block (blkcache *, uint64_t offset, uint64_t *start, uint64_t *size);
extern int put_block (blkcache *, uint64_t start, uint64_t size, char *data);
extern void blkcache_get_stats (blkcache *, blkcache_stats *ret);

#endif /* NBDKIT_XZFILE_H */

0 comments on commit c879d31

Please sign in to comment.
You can’t perform that action at this time.