Skip to content

Commit

Permalink
Jas add compression methods two (#2407)
Browse files Browse the repository at this point in the history
* Feature: add NCI attribute compression_method

Use the NCI field spare2 to hold a 1 byte compression_method which
will be used to determine which of the supported (distributed)
compression methods will be used to compress this node.

1 --> 'standard'
2 --> 'gip'

this is a work in progress.
Still need:
  tcl_set_node - keep a stack of error strings since now we have a
                 warning, which will keep going.

  tcl dir/full - display this
  tdi SETNCI and GETNCI
  treeshr (use the compression method)

* 

Use the NCI field spare2 to hold a 1 byte compression_method which
will be used to determine which of the supported (distributed)
compression methods will be used to compress this node.

1 --> 'standard'
2 --> 'gip'

this is a work in progress.
Still need:
  tcl_set_node - keep a stack of error strings since now we have a
                 warning, which will keep going.

  tcl dir/full - display this
  tdi SETNCI and GETNCI
  treeshr (use the compression method)
  • Loading branch information
joshStillerman committed Nov 1, 2021
1 parent 7f0a492 commit 9829bf6
Show file tree
Hide file tree
Showing 19 changed files with 300 additions and 47 deletions.
3 changes: 2 additions & 1 deletion .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,8 @@ camshr/cts_commands.c

# vi autosave files
*~

#vs code autosave files
*.swp
# python compiled files
*.pyc
__pycache__/
Expand Down
19 changes: 19 additions & 0 deletions _include/_ncidef.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
#ifndef INTERNAL_NCIDEF_H
#define INTERNAL_NCIDEF_H
/**************************************
_NCIDEF.H - definitions of constants
used in to define compression method
names and routines.
**************************************/


typedef struct compression_method {
char *name;
char *method;
char *image;
} COMPRESSION_METHOD;

#define DEFINE_COMPRESSION_METHODS \
static const COMPRESSION_METHOD compression_methods[] = {{"standard", NULL, NULL}, {"gzip", "gzip", "libMdsShr"}}; \
static const size_t NUM_COMPRESSION_METHODS = sizeof(compression_methods)/sizeof(COMPRESSION_METHOD);
#endif //INTERNAL_NCIDEF_H
8 changes: 5 additions & 3 deletions include/ncidef.h
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
#ifndef _NCIDEF_H
#define _NCIDEF_H
#ifndef NCIDEF_H
#define NCIDEF_H
/**************************************
NCIDEF.H - definitions of constants
used in item list arguments to the
Expand Down Expand Up @@ -38,7 +38,7 @@ TYPEDEF(4){
NciM_NID_REFERENCE = 0x00004000,
NciM_INCLUDE_IN_PULSE = 0x00008000,
NciM_COMPRESS_SEGMENTS = 0x00010000,
} ENDDEF(uint32_t, ncim_t);
} ENDDEF(uint32_t, ncim_t);
TYPEDEF(4){
NciK_IS_CHILD = 1,
NciK_IS_MEMBER = 2,
Expand Down Expand Up @@ -84,6 +84,8 @@ TYPEDEF(4){
NciUSAGE_STR = 39,
NciCLASS_STR = 40,
NciVERSION = 41,
NciCOMPRESSION_METHOD = 42,
NciCOMPRESSION_METHOD_STR = 43,
} ENDDEF(int16_t, nci_t);
#undef TYPEDEF
#undef ENDDEF
Expand Down
1 change: 1 addition & 0 deletions mdsobjects/cpp/testing/MdsTreeSegments.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -350,6 +350,7 @@ int main(int argc __attribute__((unused)),
char *argv[] __attribute__((unused)))
{
setenv("t_treeseg_path", ".", 1);
TEST_TIMEOUT(100);
TEST(putSegment);
TEST(BlockAndRows);
TEST(makeSegment);
Expand Down
1 change: 1 addition & 0 deletions mdsshr/Makefile.in
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ libShared = $(addsuffix @SHARETYPE@, $(addprefix @MAKESHLIBDIR@@LIBPRE@, $(lib_L
libs = $(libStatic) $(libShared) $(IMPLIB)

SOURCES = \
gzipcompress.c \
librtl.c \
mdsmsg.c \
MDSprintf.c \
Expand Down
54 changes: 28 additions & 26 deletions mdsshr/MdsCompress.c
Original file line number Diff line number Diff line change
Expand Up @@ -189,42 +189,44 @@ static int compress(const mdsdsc_t *const pcimage,
pdat->arsize = (unsigned int)(plim - pcmp);

nitems = (int)porig->arsize / (int)porig->length;
if (pcentry)
if (pcentry && pcentry->length && pcentry->pointer)
{
dximage = EMPTY_D;
dxentry = EMPTY_D;
status = LibFindImageSymbol(pcimage, pcentry, &symbol);
if (STATUS_OK)
status = (*symbol)(&nitems, pwork, pdat, &bit, &dximage, &dxentry);
pdat->arsize = (bit + 7) / 8;
pd0 = (mdsdsc_t *)(pdat->pointer + pdat->arsize);
if (dximage.pointer)
{
pd1 = &pd0[1] + dximage.length;
if ((char *)pd1 < (char *)plim)
status = (*symbol)(&nitems, pwork, pdat, &bit, &dximage, &dxentry);
pdat->arsize = (bit + 7) / 8;
pd0 = (mdsdsc_t *)(pdat->pointer + pdat->arsize);
if (dximage.pointer)
{
prec->dscptrs[0] = pd0;
*pd0 = *(mdsdsc_t *)&dximage;
pd0->pointer = (char *)&pd0[1];
memcpy(pd0->pointer, dximage.pointer,
dximage.length);
pd1 = &pd0[1] + dximage.length;
if ((char *)pd1 < (char *)plim)
{
prec->dscptrs[0] = pd0;
*pd0 = *(mdsdsc_t *)&dximage;
pd0->pointer = (char *)&pd0[1];
memcpy(pd0->pointer, dximage.pointer,
dximage.length);
}
pd0 = pd1;
StrFree1Dx(&dximage);
}
pd0 = pd1;
StrFree1Dx(&dximage);
}
if (dxentry.pointer)
{
pd1 = &pd0[1] + dxentry.length;
if ((char *)pd1 < (char *)plim)
if (dxentry.pointer)
{
prec->dscptrs[1] = pd0;
*pd0 = *(mdsdsc_t *)&dxentry;
pd0->pointer = (char *)&pd0[1];
memcpy(pd0->pointer, dxentry.pointer,
dxentry.length);
pd1 = &pd0[1] + dxentry.length;
if ((char *)pd1 < (char *)plim)
{
prec->dscptrs[1] = pd0;
*pd0 = *(mdsdsc_t *)&dxentry;
pd0->pointer = (char *)&pd0[1];
memcpy(pd0->pointer, dxentry.pointer,
dxentry.length);
}
pd0 = pd1;
StrFree1Dx(&dxentry);
}
pd0 = pd1;
StrFree1Dx(&dxentry);
}
if ((STATUS_OK) && (status != LibSTRTRU) &&
((char *)pd0 < (char *)plim))
Expand Down
21 changes: 18 additions & 3 deletions mdsshr/MdsSerialize.c
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#include <mdsshr.h>
#include <mdsshr_messages.h>
#include <mdstypes.h>
#include <_ncidef.h>
#include <stdint.h>
#include <string.h>

Expand All @@ -52,6 +53,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#define FixLength(a) \
if (a.length == 0) \
MdsFixDscLength((mdsdsc_t *)&a)
#define N_ELEMENTS(a) (sizeof(a) / sizeof(a[0]))
void MdsFixDscLength(mdsdsc_t *in);

#ifdef WORDS_BIGENDIAN
Expand Down Expand Up @@ -1127,12 +1129,25 @@ EXPORT int MdsSerializeDscOutZ(
MdsCopyDxXdZ(in, out, 0, fixupNid, fixupNidArg, fixupPath, fixupPathArg);
if (status == MdsCOMPRESSIBLE)
{
if (compress)
// changed the meaning of the compress arg from boolean to:
// -1 -don't compress
// 0 - use standard delta compression
// 1.. - use the compression method with this index
if (compress != -1)
{
DEFINE_COMPRESSION_METHODS

tempxd = *out;
out->l_length = 0;
out->pointer = 0;
status = MdsCompress(0, 0, tempxd.pointer, out);
if ((unsigned int)compress >= NUM_COMPRESSION_METHODS)
compress = 0;
DESCRIPTOR(image, compression_methods[compress].image);
DESCRIPTOR(method, compression_methods[compress].method);
status = MdsCompress((compress) ? &image : NULL,
(compress) ? &method : NULL,
tempxd.pointer,
out);
MdsFree1Dx(&tempxd, NULL);
compressible = 0;
}
Expand Down Expand Up @@ -1219,5 +1234,5 @@ EXPORT int MdsSerializeDscOutZ(

EXPORT int MdsSerializeDscOut(mdsdsc_t const *in, mdsdsc_xd_t *out)
{
return MdsSerializeDscOutZ(in, out, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0);
return MdsSerializeDscOutZ(in, out, 0, 0, 0, 0, -1, 0, 0, 0, 0, 0, 0, 0, 0);
}
86 changes: 86 additions & 0 deletions mdsshr/gzipcompress.c
Original file line number Diff line number Diff line change
@@ -0,0 +1,86 @@
#include <string.h>

#include <mdsdescrip.h>
#include <strroutines.h>

#include <stdio.h>
#include <stdarg.h>
#include <zlib.h>

//#define DEBUG

#define UNUSED(x) (void)(x)

EXPORT int gzip(
const int *const nitems_ptr,
const mdsdsc_a_t *const items_dsc_ptr,
mdsdsc_a_t *const pack_dsc_ptr,
int *const bit_ptr,
mdsdsc_d_t * const pdximage,
mdsdsc_d_t * const pdxentry
)
{
int ret;
UNUSED(nitems_ptr);

// unsigned long maxDestinationSize = compressBound(items_dsc_ptr->length);

unsigned long pack_length = pack_dsc_ptr->arsize;
static const DESCRIPTOR(image, "libMdsShr");
static const DESCRIPTOR(routine, "gunzip");
ret = compress(
(Bytef *)pack_dsc_ptr->pointer,
&pack_length,
(Bytef *)items_dsc_ptr->pointer,
items_dsc_ptr->arsize
);

if (ret != Z_OK) {
return LibSTRTRU;
}

#ifdef DEBUG
printf("gzip() %u => %lu\n", pack_dsc_ptr->arsize, pack_length);
#endif
// The new compressed length, in bits (for some reason)
*bit_ptr = pack_length * 8;

if (pdximage) {
StrCopyDx((mdsdsc_t * const)pdximage, &image);
}

if (pdxentry) {
StrCopyDx((mdsdsc_t * const)pdxentry, &routine);
}
#ifdef DEBUG
printf("gzip() called successfully\n");
#endif
return 1;
}
EXPORT int gunzip(
int *const nitems_ptr,
const mdsdsc_a_t *const pack_dsc_ptr,
mdsdsc_a_t *const items_dsc_ptr,
int *const bit_ptr
)
{
int ret;
UNUSED(nitems_ptr);
UNUSED(bit_ptr);
unsigned long items_length = items_dsc_ptr->arsize;

ret = uncompress(
(Bytef *)items_dsc_ptr->pointer,
&items_length,
(Bytef *)pack_dsc_ptr->pointer,
pack_dsc_ptr->arsize
);

if (ret != Z_OK) {
return LibINVSTRDES;
}
#ifdef DEBUG
printf("gunzip() called successfully\n");
#endif
return 1;
}
41 changes: 41 additions & 0 deletions python/MDSplus/tree.py
Original file line number Diff line number Diff line change
Expand Up @@ -90,6 +90,14 @@ def __init__(self, usage):
super(UsageError, self).__init__('Invalid usage "%s". Must be one of: %s' % (
str(usage), ', '.join(_usage_table.keys())))

_compression_methods_table = {'standard': 0, 'gzip': 1}

class CompressionMethodError(KeyError):
def __init__(self, compression_method):
super(CompressionMethodError, self).__init__('Invalid compression_method "%s". Must be one of: %s' % (
str(compression_method), ', '.join(_compression_methods_table.keys())))


#
###################################################

Expand Down Expand Up @@ -300,6 +308,8 @@ class Nci(object):
USAGE_STR = (39, _C.c_char_p, 64, str)
CLASS_STR = (40, _C.c_char_p, 64, str)
VERSION = (41, _C.c_uint32, 4, int)
COMPRESSION_METHOD = (42, _C.c_uint8, 1, int)
COMPRESSION_METHOD_STR = (43, _C.c_char_p, 64, str)

class _nci_item(_C.Structure):
_fields_ = [("buflen", _C.c_ushort),
Expand Down Expand Up @@ -1308,6 +1318,11 @@ def _getNci(self, info):
return TreeNodeArray([int(ans[i]) for i in _ver.xrange(retlen//4)], self.tree)
return rtype(ans.value)

def _setNci(self, code, setting):
pointer = _C.cast(_C.pointer(_C.c_uint32(setting)), _C.c_char_p)
_exc.checkStatus(_TreeShr._TreeSetNci(
self.ctx, self._nid, _C.byref(Nci._nci_item(0, code, pointer))))

def _setNciFlag(self, mask, setting):
value = 1 if setting else 2
pointer = _C.cast(_C.pointer(_C.c_uint32(mask)), _C.c_char_p)
Expand Down Expand Up @@ -1394,6 +1409,11 @@ def _setNciFlag(self, mask, setting):
Nci.TIME_INSERTED, "64-bit timestamp when data was stored")
usage_str = Nci._nciProp(
Nci.USAGE_STR, "formal name of the usage of this node")
compression_method = Nci._nciProp(
Nci.COMPRESSION_METHOD, "numerical code for the compression method to use for this node")
compression_method_str = Nci._nciProp(
Nci.COMPRESSION_METHOD_STR, "name of the compression algorithm to use for this node")

__children_nids = Nci._nciProp(Nci.CHILDREN_NIDS)
__member_nids = Nci._nciProp(Nci.MEMBER_NIDS)

Expand Down Expand Up @@ -1549,6 +1569,15 @@ def usage(self):
@usage.setter
def usage(self, usage): self.setUsage(usage)

@property
def compression_method(self):
"compression method to use for this node."
return _scr.String(str(self.compression_method_str))

@compression_method.setter
def compression_method(self, compression_method):
self.setCompressionMethod(compression_method)

########################################
# End of Node Properties
########################################
Expand Down Expand Up @@ -2938,6 +2967,18 @@ def setUsage(self, usage):
_C.c_int32(usagenum)))
return self

def setCompressionMethod(self, compression_method):
"""Set the usage of a node
@param compression_method: name of compression method from predfined list.
@type usage: str
@rtype: original type
"""
try:
compressionmethodnum = _compression_methods_table[compression_method.lower()]
except KeyError:
raise CompressionMethodError(compression_method)
self._setNci(Nci.COMPRESSION_METHOD[0], compressionmethodnum)

def setTree(self, tree):
"""Set Tree associated with this node
@param tree: Tree instance to associated with this node
Expand Down
Loading

0 comments on commit 9829bf6

Please sign in to comment.