changes for python3 modules, and fixes so the deployment tests work (#89

)

* changes for python3 modules, and fixes so the deployment tests work

* cleanup bike-sheds, and remove debug remnants

pacifica/archiveinterface/backends/hpss/archive.py

@@ -4,7 +4,7 @@
 import os
 import sys
 from ctypes import cdll, create_string_buffer, cast
-from ctypes import c_void_p, c_char_p, c_int, c_size_t
+from ctypes import c_void_p, c_char_p, c_int, c_size_t, c_long
 from ...archive_utils import un_abs_path
 from ...config import get_config
 from ...exception import ArchiveInterfaceError
@@ -46,6 +46,10 @@
 HPSS_RPC_AUTH_TYPE_KEY = 4
 HPSS_RPC_AUTH_TYPE_PASSWD = 5
 
+SEEK_SET = 0
+SEEK_CUR = 1
+SEEK_END = 2
+
 
 def path_info_munge(filepath):
     """Munge the path for this filetype."""
@@ -96,13 +100,24 @@ def open(self, filepath, mode):
             hpss.ping_core(self._sitename)
             hpss.makedirs()
             hpss_fopen = self._hpsslib.hpss_Fopen
-            hpss_fopen.restype = c_void_p
+            hpss_fopen.restype = c_long
             hpss_fopen.argtypes = [c_char_p, c_char_p]
-            self._file = hpss_fopen(filename, mode)
+            self._file = hpss_fopen(filename.encode('utf8'), mode.encode('utf8'))
             self._check_rcode(
                 self._file,
                 'Failed opening Hpss File, code: ' + str(self._file)
             )
+            if self._file == 0:
+                raise ArchiveInterfaceError('NULL File returned on open')
+
+            # this stops a race where open seems to start a read async,
+            # and then if you delete the file we get a sigabort
+            # seeking throws away all the buffers..
+            hpss_fseek = self._hpsslib.hpss_Fseek
+            hpss_fseek.restype = c_long
+            hpss_fseek.argtypes = [c_void_p, c_long, c_int]
+            hpss_fseek(self._file, SEEK_SET, 0)
+
             return self
         except Exception as ex:
             err_str = "Can't open hpss file with error: " + str(ex)
@@ -139,7 +154,7 @@ def read(self, blocksize):
         """Read a file from the hpss archive."""
         try:
             if self._filepath:
-                buf = create_string_buffer('\000' * blocksize)
+                buf = create_string_buffer(blocksize)
                 hpss_fread = self._hpsslib.hpss_Fread
                 hpss_fread.restype = c_size_t
                 hpss_fread.argtypes = [c_void_p, c_size_t, c_size_t, c_void_p]
@@ -211,7 +226,7 @@ def set_file_permissions(self):
             if self._filepath:
                 hpss = HpssExtended(self._filepath, self._latency)
                 hpss.ping_core(self._sitename)
-                rcode = self._hpsslib.hpss_Chmod(self._filepath, 0o444)
+                rcode = self._hpsslib.hpss_Chmod(self._filepath.encode('utf8'), 0o444)
                 self._check_rcode(
                     rcode,
                     'Failed to chmod hpss file with code: ' + str(rcode)
@@ -225,12 +240,12 @@ def authenticate(self):
         user = get_config().get('hpss', 'user')
         auth = get_config().get('hpss', 'auth')
         rcode = self._hpsslib.hpss_SetLoginCred(
-            user, HPSS_AUTHN_MECH_UNIX,
-            HPSS_RPC_CRED_CLIENT, HPSS_RPC_AUTH_TYPE_KEYTAB, auth
+            user.encode('utf8'), HPSS_AUTHN_MECH_UNIX,
+            HPSS_RPC_CRED_CLIENT, HPSS_RPC_AUTH_TYPE_KEYTAB, auth.encode('utf8')
         )
         self._check_rcode(
             rcode,
-            'Could Not Authenticate, error code is:' + str(rcode)
+            'Could Not Authenticate, error code is:' + str(rcode) + ' User: ' + user + ' Auth: ' + auth
         )
 
     def patch(self, file_id, old_path):
@@ -259,11 +274,14 @@ def remove(self):
         """Remove the file for an HPSS file."""
         try:
             if self._filepath:
-                buf_char_p = cast(self._filepath, c_char_p)
+                buf_char_p = cast(self._filepath.encode(), c_char_p)
                 rcode = self._hpsslib.hpss_Chmod(buf_char_p, 0o644)
+                self._check_rcode(rcode, 'Error chmoding hpss file: {}'.format(rcode))
+
                 hpss_unlink = self._hpsslib.hpss_Unlink
                 hpss_unlink.restype = c_int
                 hpss_unlink.argtypes = [c_char_p]
+                buf_char_p = cast(self._filepath.encode(), c_char_p)
                 rcode = hpss_unlink(buf_char_p)
                 self._check_rcode(rcode, 'Error removing hpss file: {}'.format(rcode))
             self._filepath = None

pacifica/archiveinterface/backends/hpss/hpssExtensions.c

@@ -3,6 +3,7 @@
 #include <time.h>
 #include <stdlib.h>
 #include <libgen.h>
+#include "hpss_version.h"
 #include "hpss_api.h"
 #include <sys/types.h>
 #include <utime.h>
@@ -323,7 +324,28 @@ pacifica_archiveinterface_makedirs(PyObject *self, PyObject *args)
     return rec_makedirs(filepath);
 }
 
-static PyMethodDef StatusMethods[] = {
+/* python 2 & 3 support from https://docs.python.org/3/howto/cporting.html */
+
+struct module_state {
+    PyObject *error;
+};
+
+#if PY_MAJOR_VERSION >= 3
+#define GETSTATE(m) ((struct module_state*)PyModule_GetState(m))
+#else
+#define GETSTATE(m) (&_state)
+static struct module_state _state;
+#endif
+
+/*
+static PyObject *
+error_out(PyObject *m) {
+    struct module_state *st = GETSTATE(m);
+    PyErr_SetString(st->error, "something bad happened");
+    return NULL;
+}
+*/
+static PyMethodDef _hpssExtensions_methods[] = {
     {"hpss_status", pacifica_archiveinterface_status, METH_VARARGS,
         "Get the status for a file in the archive."},
     {"hpss_mtime", pacifica_archiveinterface_mtime, METH_VARARGS,
@@ -343,17 +365,65 @@ static PyMethodDef StatusMethods[] = {
     {NULL, NULL, 0, NULL}        /* Sentinel */
 };
 
+#if PY_MAJOR_VERSION >= 3
+
+static int _hpssExtensions_traverse(PyObject *m, visitproc visit, void *arg) {
+    Py_VISIT(GETSTATE(m)->error);
+    return 0;
+}
+
+static int _hpssExtensions_clear(PyObject *m) {
+    Py_CLEAR(GETSTATE(m)->error);
+    return 0;
+}
+
+
+static struct PyModuleDef moduledef = {
+        PyModuleDef_HEAD_INIT,
+        "_hpssExtensions",
+        NULL,
+        sizeof(struct module_state),
+        _hpssExtensions_methods,
+        NULL,
+        _hpssExtensions_traverse,
+        _hpssExtensions_clear,
+        NULL
+};
+
+#define INITERROR return NULL
+
 PyMODINIT_FUNC
-init_hpssExtensions(void)
+PyInit__hpssExtensions(void)
+
+#else
+#define INITERROR return
+
+void
+init__hpssExtensions(void)
+#endif
 {
-    PyObject * m;
-    m = Py_InitModule("_hpssExtensions", StatusMethods);
-    if (m == NULL)
-    {
-        return;
+#if PY_MAJOR_VERSION >= 3
+    PyObject *module = PyModule_Create(&moduledef);
+#else
+    PyObject *module = Py_InitModule("_hpssExtensions", _hpssExtensions_methods);
+#endif
+
+    if (module == NULL)
+        INITERROR;
+    struct module_state *st = GETSTATE(module);
+
+    st->error = PyErr_NewException("_hpssExtensions.Error", NULL, NULL);
+    if (st->error == NULL) {
+        Py_DECREF(module);
+        INITERROR;
     }
 
+
     archiveInterfaceError = PyErr_NewException("archiveInterface.error", NULL, NULL);
     Py_INCREF(archiveInterfaceError);
-    PyModule_AddObject(m,"error",archiveInterfaceError);
+    PyModule_AddObject(module,"error",archiveInterfaceError);
+
+#if PY_MAJOR_VERSION >= 3
+    return module;
+#endif
 }