diff --git a/LICENSE b/LICENSE index f661a01..67d7ea5 100755 --- a/LICENSE +++ b/LICENSE @@ -1,21 +1,29 @@ -MIT License +BSD 3-Clause License -Copyright (c) 2022 Advanced Bioimaging Center +Copyright (c) 2022, Advanced Bioimaging Center +All rights reserved. -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. +1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/README.md b/README.md index 07c2113..bc2154d 100755 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# c-zarr +# cpp-zarr An efficient parallel Zarr reader/writer that utilizes c-blosc/c-blosc2 and OpenMP. ## Quick Start Guide (MATLAB) @@ -7,7 +7,7 @@ An efficient parallel Zarr reader/writer that utilizes c-blosc/c-blosc2 and Open 1. All neccessary libraries are included for the Linux and Windows version. ### Download and Install -1. Download the latest release for your OS from here (windows.zip/linux.tar.gz): https://github.com/abcucberkeley/c-zarr/releases +1. Download the latest release for your OS from here (windowsMatlab.zip/linuxMatlab.tar.gz): https://github.com/abcucberkeley/cpp-zarr/releases 2. Unzip the folder 3. You can now put the folders wherever you'd like and add them to your path if needed. Just keep the mex files with their associated dll files so the mex function can always run. diff --git a/createZarrFile/createZarrFile.c b/createZarrFile/createZarrFile.c deleted file mode 100755 index 22c7718..0000000 --- a/createZarrFile/createZarrFile.c +++ /dev/null @@ -1,368 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include -#include -#ifdef _WIN32 -#include -#include -#else -#include -#endif -#include -#include "mex.h" - -//compile -//mex -v COPTIMFLAGS="-O3 -DNDEBUG" LDOPTIMFLAGS="-Wl',-rpath='''$ORIGIN'''' -O3 -DNDEBUG" CFLAGS='$CFLAGS -O3 -fopenmp' LDFLAGS='$LDFLAGS -O3 -fopenmp' '-I/global/home/groups/software/sl-7.x86_64/modules/cJSON/1.7.15/include/' -L'/global/home/groups/software/sl-7.x86_64/modules/cJSON/1.7.15/lib64' -lcjson -luuid createZarrFile.c - -// Handle the tilde character in filenames on Linux/Mac -#ifndef _WIN32 -#include -char* expandTilde(char* path) { - wordexp_t expPath; - wordexp(path, &expPath, 0); - return expPath.we_wordv[0]; -} -#endif - -const char fileSep = -#ifdef _WIN32 - '\\'; -#else - '/'; -#endif - -#ifdef _WIN32 -char* strndup (const char *s, size_t n) -{ - size_t len = strnlen (s, n); - char *new = (char *) malloc (len + 1); - if (new == NULL) - return NULL; - new[len] = '\0'; - return (char *) memcpy (new, s, len); -} -int _vscprintf_so(const char * format, va_list pargs) { - int retval; - va_list argcopy; - va_copy(argcopy, pargs); - retval = vsnprintf(NULL, 0, format, argcopy); - va_end(argcopy); - return retval; -} - -int vasprintf(char **strp, const char *fmt, va_list ap) { - int len = _vscprintf_so(fmt, ap); - if (len == -1) return -1; - char *str = malloc((size_t) len + 1); - if (!str) return -1; - int r = vsnprintf(str, len + 1, fmt, ap); /* "secure" version of vsprintf */ - if (r == -1) return free(str), -1; - *strp = str; - return r; -} - -int asprintf(char *strp[], const char *fmt, ...) { - va_list ap; - va_start(ap, fmt); - int r = vasprintf(strp, fmt, ap); - va_end(ap); - return r; -} -#endif - -static void mkdirRecursive(const char *dir) { - char tmp[8192]; - char *p = NULL; - size_t len; - #ifdef _WIN32 - char fileSep = '\\'; - #else - char fileSep = '/'; - #endif - int status; - snprintf(tmp, sizeof(tmp),"%s",dir); - len = strlen(tmp); - if (tmp[len - 1] == fileSep) - tmp[len - 1] = 0; - for (p = tmp + 1; *p; p++){ - if (*p == fileSep) { - *p = 0; - - #ifdef _WIN32 - mkdir(tmp); - #else - mkdir(tmp, 0775); - #endif - - chmod(tmp, 0775); - *p = fileSep; - } - } - #ifdef _WIN32 - mkdir(tmp); - #else - mkdir(tmp, 0775); - #endif - chmod(tmp, 0775); -} - -void setJSONValues(char* fileName,uint64_t *chunkXSize,uint64_t *chunkYSize,uint64_t *chunkZSize,char* dtype,char* order,uint64_t *shapeX,uint64_t *shapeY,uint64_t *shapeZ, char* cname, uint64_t level, uint64_t *subfolderSizeX, uint64_t *subfolderSizeY, uint64_t *subfolderSizeZ){ - - // Overflows for ints greater than 32 bit for chunkSizes and shape - cJSON* zArray = cJSON_CreateObject(); - const int chunkSizes[3] = {*chunkXSize,*chunkYSize,*chunkZSize}; - cJSON* chunks = cJSON_CreateIntArray(chunkSizes, 3); - cJSON_AddItemToObject(zArray, "chunks", chunks); - - cJSON* compressor = cJSON_CreateObject(); - cJSON_AddItemToObject(zArray, "compressor", compressor); - if(!strcmp(cname,"lz4") || !strcmp(cname,"blosclz") || !strcmp(cname,"lz4hc") || !strcmp(cname,"zlib") || !strcmp(cname,"zstd")){ - cJSON_AddNumberToObject(compressor, "blocksize", 0); - cJSON_AddNumberToObject(compressor, "clevel", level); - cJSON_AddStringToObject(compressor, "cname", cname); - cJSON_AddStringToObject(compressor, "id", "blosc"); - cJSON_AddNumberToObject(compressor, "shuffle", 1); - } - else if(!strcmp(cname,"gzip")){ - cJSON_AddStringToObject(compressor, "id", cname); - cJSON_AddNumberToObject(compressor, "level", level); - } - else mexErrMsgIdAndTxt("zarr:zarrayError","Compressor: \"%s\" is not currently supported\n",cname); - - cJSON_AddStringToObject(zArray, "dtype", dtype); - cJSON_AddNumberToObject(zArray, "fill_value", 0); - cJSON_AddNullToObject(zArray, "filters"); - cJSON_AddStringToObject(zArray, "order", order); - - const int shapeSizes[3] = {*shapeX,*shapeY,*shapeZ}; - cJSON* shape = cJSON_CreateIntArray(shapeSizes, 3); - cJSON_AddItemToObject(zArray, "shape", shape); - cJSON_AddNumberToObject(zArray, "zarr_format", 2); - - const int subfolderSizeSizes[3] = {*subfolderSizeX,*subfolderSizeY,*subfolderSizeZ}; - cJSON* subfolderSize = cJSON_CreateIntArray(subfolderSizeSizes, 3); - cJSON_AddItemToObject(zArray, "subfolders", subfolderSize); - - uint64_t uuidLen; - #ifdef _WIN32 - uuidLen = 5; - char *uuid = malloc(uuidLen+1); - char *seedArr = malloc(1000); - struct timeval cSeed; - gettimeofday(&cSeed,NULL); - int nChars = sprintf(seedArr,"%d%d",cSeed.tv_sec,cSeed.tv_usec); - int aSeed = 0; - char* ptr; - if(nChars > 9) - aSeed = strtol(seedArr+nChars-10, &ptr, 9); - else aSeed = strtol(seedArr, &ptr, 9); - srand(aSeed); - sprintf(uuid,"%.5d",rand() % 99999); - #else - uuidLen = 36; - uuid_t binuuid; - uuid_generate_random(binuuid); - char *uuid = malloc(uuidLen+1); - uuid_unparse(binuuid, uuid); - #endif - - char* zArrayS = ".zarray"; - char* fnFull = (char*)malloc(strlen(fileName)+8+uuidLen+1); - fnFull[0] = '\0'; - char fileSepS[2]; - fileSepS[0] = fileSep; - fileSepS[1] = '\0'; - - strcat(fnFull,fileName); - strcat(fnFull,fileSepS); - strcat(fnFull,zArrayS); - char* fileNameFinal = strndup(fnFull,strlen(fileName)+1+8); - strcat(fnFull,uuid); - - char* string = cJSON_Print(zArray); - - FILE *fileptr = fopen(fnFull, "w+"); - if(!fileptr) mexErrMsgIdAndTxt("zarr:zarrayError","Cannot open %s\n",fnFull); - fprintf(fileptr,"%s",string); - fclose(fileptr); - - rename(fnFull,fileNameFinal); - //file = fopen(fileNameFinal, "r"); - //if(!file) rename(fileName,fileNameFinal); - - cJSON_Delete(zArray); - free(fnFull); - free(uuid); - free(fileNameFinal); -} - -uint64_t fastCeilDiv(uint64_t num, uint64_t denom){ - return 1 + ((num - 1) / denom); -} - -void createSubfolders(char* folderName, uint64_t shapeX, uint64_t shapeY, uint64_t shapeZ, uint64_t chunkXSize, uint64_t chunkYSize, uint64_t chunkZSize, uint64_t subfolderSizeX, uint64_t subfolderSizeY, uint64_t subfolderSizeZ){ - if(subfolderSizeX == 0 && subfolderSizeY == 0 && subfolderSizeZ == 0) return; - uint64_t nChunksX = fastCeilDiv(shapeX,chunkXSize); - uint64_t nChunksY = fastCeilDiv(shapeY,chunkYSize); - uint64_t nChunksZ = fastCeilDiv(shapeZ,chunkZSize); - - uint64_t nSubfoldersX = 1; - uint64_t nSubfoldersY = 1; - uint64_t nSubfoldersZ = 1; - - if(subfolderSizeX > 0) nSubfoldersX = fastCeilDiv(nChunksX,subfolderSizeX); - if(subfolderSizeY > 0) nSubfoldersY = fastCeilDiv(nChunksY,subfolderSizeY); - if(subfolderSizeZ > 0) nSubfoldersZ = fastCeilDiv(nChunksZ,subfolderSizeZ); - - #pragma omp parallel for collapse(3) - for(uint64_t x = 0; x < nSubfoldersX; x++){ - for(uint64_t y = 0; y < nSubfoldersY; y++){ - for(uint64_t z = 0; z < nSubfoldersZ; z++){ - char* currName = NULL; - asprintf(&currName,"%s/%llu_%llu_%llu",folderName,x,y,z); - mkdirRecursive(currName); - free(currName); - } - } - } -} - -void mexFunction(int nlhs, mxArray *plhs[], - int nrhs, const mxArray *prhs[]) -{ - // Needed variables for .zarray file - uint64_t chunkXSize = 256; - uint64_t chunkYSize = 256; - uint64_t chunkZSize = 256; - char* cname = "lz4"; - char* dtype = "\"\n"); - } - - if(dtype[1] == 'u' || dtype[1] == 'f'){ - if(dtype[1] == 'u'){ - if(dtype[2] != '1' && dtype[2] != '2'){ - mexErrMsgIdAndTxt("zarr:inputError","For dtype u, the third character of dtype must be \"1\" or \"2\"\n"); - } - } - else{ - if(dtype[2] != '4' && dtype[2] != '8'){ - mexErrMsgIdAndTxt("zarr:inputError","For dtype f, the third character of dtype must be \"4\" or \"8\"\n"); - } - } - } - else mexErrMsgIdAndTxt("zarr:inputError","The second character of dtype must be \"u\" or \"f\"\n"); - } - else if(!strcmp(currInput,"order")){ - if(!mxIsChar(prhs[i+1])) mexErrMsgIdAndTxt("zarr:inputError","order must be a string\n"); - order = mxArrayToString(prhs[i+1]); - if(strlen(order) > 1) mexErrMsgIdAndTxt("zarr:inputError","order must be of length 1\n"); - if(order[0] != 'F' || order[0] != 'C' || order[0] != 'f' || order[0] != 'c'){ - if(order[0] == 'f') order = "F"; - else if(order [0] == 'c') order = "C"; - } - else mexErrMsgIdAndTxt("zarr:inputError","order must be \"F\" or \"C\"\n"); - } - else if(!strcmp(currInput,"shape")){ - if(mxGetN(prhs[i+1]) != 3) mexErrMsgIdAndTxt("zarr:inputError","shape must be an array of 3 numbers\n"); - shapeX = (uint64_t)*(mxGetPr(prhs[i+1])); - shapeY = (uint64_t)*((mxGetPr(prhs[i+1])+1)); - shapeZ = (uint64_t)*((mxGetPr(prhs[i+1])+2)); - } - else if(!strcmp(currInput,"clevel")){ - if(!mxIsNumeric(prhs[i+1])) mexErrMsgIdAndTxt("zarr:inputError","level must be a numerical value\n"); - level = (uint64_t)*(mxGetPr(prhs[i+1])); - } - else if(!strcmp(currInput,"subfolders")){ - if(mxGetN(prhs[i+1]) != 3) mexErrMsgIdAndTxt("zarr:inputError","subfolders must be an array of 3 numbers\n"); - subfolderSizeX = (uint64_t)*(mxGetPr(prhs[i+1])); - subfolderSizeY = (uint64_t)*((mxGetPr(prhs[i+1])+1)); - subfolderSizeZ = (uint64_t)*((mxGetPr(prhs[i+1])+2)); - } - else{ - mexErrMsgIdAndTxt("zarr:inputError","The argument \"%s\" does not match the name of any supported input name.\n \ - Currently Supported Names: chunks, cname, dtype, order, shape\n",currInput); - } - } - - - - - char* zArray = ".zarray"; - char* fnFull = (char*)malloc(strlen(folderName)+9); - fnFull[0] = '\0'; - char fileSepS[2]; - fileSepS[0] = '/'; - fileSepS[1] = '\0'; - - strcat(fnFull,folderName); - strcat(fnFull,fileSepS); - strcat(fnFull,zArray); - - FILE* f = fopen(fnFull,"r"); - if(f) fclose(f); - else{ - mkdirRecursive(folderName); - chmod(folderName, 0775); - } - createSubfolders(folderName,shapeX,shapeY,shapeZ,chunkXSize,chunkYSize,chunkZSize,subfolderSizeX,subfolderSizeY,subfolderSizeZ); - setJSONValues(folderName,&chunkXSize,&chunkYSize,&chunkZSize,dtype,order,&shapeX,&shapeY,&shapeZ,cname,level,&subfolderSizeX,&subfolderSizeY,&subfolderSizeZ); - - - ///////// -} diff --git a/createZarrFile/linux/createZarrFile.mexa64 b/createZarrFile/linux/createZarrFile.mexa64 deleted file mode 100755 index 91f8e1d..0000000 Binary files a/createZarrFile/linux/createZarrFile.mexa64 and /dev/null differ diff --git a/createZarrFile/linux/libcjson.so b/createZarrFile/linux/libcjson.so deleted file mode 100755 index eb168c2..0000000 Binary files a/createZarrFile/linux/libcjson.so and /dev/null differ diff --git a/createZarrFile/linux/libcjson.so.1 b/createZarrFile/linux/libcjson.so.1 deleted file mode 100755 index eb168c2..0000000 Binary files a/createZarrFile/linux/libcjson.so.1 and /dev/null differ diff --git a/createZarrFile/linux/libcjson.so.1.7.15 b/createZarrFile/linux/libcjson.so.1.7.15 deleted file mode 100755 index eb168c2..0000000 Binary files a/createZarrFile/linux/libcjson.so.1.7.15 and /dev/null differ diff --git a/createZarrFile/linux/libgomp.so b/createZarrFile/linux/libgomp.so deleted file mode 100755 index eccfabc..0000000 Binary files a/createZarrFile/linux/libgomp.so and /dev/null differ diff --git a/createZarrFile/linux/libgomp.so.1 b/createZarrFile/linux/libgomp.so.1 deleted file mode 100755 index eccfabc..0000000 Binary files a/createZarrFile/linux/libgomp.so.1 and /dev/null differ diff --git a/createZarrFile/linux/libgomp.so.1.0.0 b/createZarrFile/linux/libgomp.so.1.0.0 deleted file mode 100755 index eccfabc..0000000 Binary files a/createZarrFile/linux/libgomp.so.1.0.0 and /dev/null differ diff --git a/createZarrFile/windows/cjson.dll b/createZarrFile/windows/cjson.dll deleted file mode 100755 index 4e3efdc..0000000 Binary files a/createZarrFile/windows/cjson.dll and /dev/null differ diff --git a/createZarrFile/windows/createZarrFile.mexw64 b/createZarrFile/windows/createZarrFile.mexw64 deleted file mode 100755 index 72db184..0000000 Binary files a/createZarrFile/windows/createZarrFile.mexw64 and /dev/null differ diff --git a/mexSrc/compile_createZarrFile.m b/mexSrc/compile_createZarrFile.m new file mode 100644 index 0000000..28d8d7e --- /dev/null +++ b/mexSrc/compile_createZarrFile.m @@ -0,0 +1,36 @@ +% ADD --disable-new-dtags +if isunix && ~ismac + mex -v CXXOPTIMFLAGS="-O2 -DNDEBUG" LDOPTIMFLAGS="-Wl',-rpath='''$ORIGIN'''' -O2 -DNDEBUG" CXXFLAGS='$CXXFLAGS -O2 -fopenmp' LDFLAGS='$LDFLAGS -O2 -fopenmp' -I'/clusterfs/fiona/matthewmueller/cppZarrTest' -luuid createzarrfile.cpp helperfunctions.cpp zarr.cpp + % Need to change the library name because matlab preloads their own version + % of libstdc++ + % Setting it to libstdc++.so.6.0.30 as of MATLAB R2022b + system('patchelf --replace-needed libstdc++.so.6 libstdc++.so.6.0.30 createzarrfile.mexa64'); + %system('export LD_LIBRARY_PATH="";patchelf --replace-needed libstdc++.so.6 libstdc++.so.6.0.30 createzarrfile.mexa64'); + mkdir('../cpp-zarr_linux'); + movefile('createzarrfile.mexa64','../cpp-zarr_linux/createZarrFile.mexa64'); +elseif ismac + % Might have to do this part in terminal. First change the library + % linked to libstdc++ + % system('install_name_tool -change @rpath/libgcc_s.1.1.dylib @loader_path/libgcc_s.1.1.0.dylib ../cpp-zarr_mac/libstdc++.6.0.30.dylib'); + + %mex -v CXX="/usr/local/bin/g++-12" CXXOPTIMFLAGS='-O2 -DNDEBUG' LDOPTIMFLAGS='-O2 -DNDEBUG' CXXFLAGS='-fno-common -arch x86_64 -mmacosx-version-min=10.15 -fexceptions -isysroot /Library/Developer/CommandLineTools/SDKs/MacOSX.sdk -std=c++11 -O2 -fopenmp -DMATLAB_DEFAULT_RELEASE=R2017b -DUSE_MEX_CMD -DMATLAB_MEX_FILE' LDFLAGS='$LDFLAGS -O2 -fopenmp' '-I/usr/local/include/' -lstdc++ -luuid createzarrfile.cpp helperfunctions.cpp zarr.cpp + mex -v CXX="/usr/local/bin/g++-12" CXXOPTIMFLAGS='-O2 -DNDEBUG' LDOPTIMFLAGS='-O2 -DNDEBUG' CXXFLAGS='-fno-common -arch arm64 -mmacosx-version-min=10.15 -fexceptions -isysroot /Library/Developer/CommandLineTools/SDKs/MacOSX.sdk -std=c++11 -O2 -fopenmp -DMATLAB_DEFAULT_RELEASE=R2017b -DUSE_MEX_CMD -DMATLAB_MEX_FILE' LDFLAGS='$LDFLAGS -O2 -fopenmp' '-I/usr/local/include/' -lstdc++ -luuid createzarrfile.cpp helperfunctions.cpp zarr.cpp + %mex -v CXX='/usr/local/bin/g++-12' CXXOPTIMFLAGS='-O2 -DNDEBUG' LDOPTIMFLAGS='-O2 -DNDEBUG' CXXFLAGS='$CXXFLAGS -O2 -fopenmp' LDFLAGS='$LDFLAGS -O2 -fopenmp' '-I/usr/local/include/' -L'/usr/local/lib/' -luuid createZarrFile.cpp + %mex -v CXX="/usr/local/opt/llvm/bin/clang++" CXXOPTIMFLAGS='-O2 -DNDEBUG' LDOPTIMFLAGS='-O2 -DNDEBUG' CXXFLAGS='$CXXFLAGS -O2 -fopenmp' LDFLAGS='$LDFLAGS -O2 -fopenmp' '-I/usr/local/include/' -L'/usr/local/lib/' -luuid createZarrFile.cpp helperfunctions.cpp zarr.cpp + + % We need to change all the current paths to be relative to the mex file + system('install_name_tool -change /usr/local/opt/gcc@12/lib/gcc/12/libstdc++.6.dylib @loader_path/libstdc++.6.0.30.dylib createzarrfile.mexmaci64'); + system('install_name_tool -change /usr/local/opt/gcc@12/lib/gcc/12/libgcc_s.1.1.dylib @loader_path/libgcc_s.1.1.0.dylib createzarrfile.mexmaci64'); + system('install_name_tool -change /usr/local/opt/ossp-uuid/lib/libuuid.16.dylib @loader_path/libuuid.16.22.0.dylib createzarrfile.mexmaci64'); + system('install_name_tool -change /usr/local/opt/gcc@12/lib/gcc/12/libgomp.1.dylib @loader_path/libgomp.1.dylib createzarrfile.mexmaci64'); + + + system('chmod 777 createzarrfile.mexmaci64'); + mkdir('../cpp-zarr_mac'); + movefile('createzarrfile.mexmaci64','../cpp-zarr_mac/createZarrFile.mexmaci64'); +elseif ispc + mex CXX="C:/mingw64/bin/g++" -v CXXOPTIMFLAGS="-DNDEBUG -O2" LDOPTIMFLAGS="-Wl',-rpath='''$ORIGIN'''' -O2 -DNDEBUG" CXXFLAGS='$CXXFLAGS -fopenmp -O2' LDFLAGS='$LDFLAGS -fopenmp -O2' -I'C:/Program Files (x86)/nlohmann_json/include' createzarrfile.cpp helperfunctions.cpp zarr.cpp + + mkdir('../cpp-zarr_windows'); + movefile('createzarrfile.mexw64','../cpp-zarr_windows/createZarrFile.mexw64'); +end \ No newline at end of file diff --git a/mexSrc/compile_parallelReadZarr.m b/mexSrc/compile_parallelReadZarr.m new file mode 100644 index 0000000..585c8b6 --- /dev/null +++ b/mexSrc/compile_parallelReadZarr.m @@ -0,0 +1,51 @@ +debug = false; +% ADD --disable-new-dtags +if isunix && ~ismac + if debug + %mex -v -g CXXOPTIMFLAGS="" LDOPTIMFLAGS="-Wl',-rpath='''$ORIGIN''''" CXXFLAGS='$CXXFLAGS -fopenmp' LDFLAGS='$LDFLAGS -fopenmp -O2' -I'/clusterfs/fiona/matthewmueller/cppZarrTest' -I'/global/home/groups/software/sl-7.x86_64/modules/cBlosc/2.0.4/include/' '-L/global/home/groups/software/sl-7.x86_64/modules/cBlosc/2.0.4/lib64' -lblosc2 -lz -luuid parallelreadzarr.cpp zarr.cpp helperfunctions.cpp parallelreadzarrwrapper.cpp + mex -v -g CXXOPTIMFLAGS="" LDOPTIMFLAGS="-g -O0 -Wall -Wextra -Wl',-rpath='''$ORIGIN''''" CXXFLAGS='$CXXFLAGS -Wall -Wextra -g -O0 -fsanitize=address -gdwarf-4 -gstrict-dwarf' LDFLAGS='$LDFLAGS -g -O0 -fopenmp -fsanitize=address' -I'/clusterfs/fiona/matthewmueller/cppZarrTest' -I'/global/home/groups/software/sl-7.x86_64/modules/cBlosc/2.8.0/include/' '-L/global/home/groups/software/sl-7.x86_64/modules/cBlosc/2.8.0/lib64' -lblosc2 -lz -luuid parallelreadzarr.cpp zarr.cpp helperfunctions.cpp parallelreadzarrwrapper.cpp + %mex -v -g CXXOPTIMFLAGS="" LDOPTIMFLAGS="-g -O0 -Wall -Wextra -Wl',-rpath='''$ORIGIN''''" CXXFLAGS='$CXXFLAGS -Wall -Wextra -g -O0 -fsanitize=address' LDFLAGS='$LDFLAGS -g -O0 -fopenmp -fsanitize=address' -I'/clusterfs/fiona/matthewmueller/cppZarrTest' -I'/global/home/groups/software/sl-7.x86_64/modules/cBlosc/2.8.0/include/' '-L/global/home/groups/software/sl-7.x86_64/modules/cBlosc/2.8.0/lib64' -lblosc2 -lz -luuid parallelreadzarr.cpp zarr.cpp helperfunctions.cpp parallelreadzarrwrapper.cpp + else + %mex -v CXXOPTIMFLAGS="-DNDEBUG -O2" LDOPTIMFLAGS="-Wl',-rpath='''$ORIGIN'''' -O2 -DNDEBUG" CXXFLAGS='$CXXFLAGS -fopenmp -O2' LDFLAGS='$LDFLAGS -fopenmp -O2' -I'/clusterfs/fiona/matthewmueller/cppZarrTest' -I'/global/home/groups/software/sl-7.x86_64/modules/cBlosc/2.3.1/include/' '-L/global/home/groups/software/sl-7.x86_64/modules/cBlosc/2.3.1/lib64' -lblosc2 -lz -luuid parallelreadzarr.cpp zarr.cpp helperfunctions.cpp parallelreadzarrwrapper.cpp + %mex -v CXXOPTIMFLAGS="-DNDEBUG -O2" LDOPTIMFLAGS="-Wl',-rpath='''$ORIGIN'''' -O2 -DNDEBUG" CXXFLAGS='$CXXFLAGS -fopenmp -O2' LDFLAGS='$LDFLAGS -fopenmp -O2' -I'/clusterfs/fiona/matthewmueller/cppZarrTest' -I'/global/home/groups/software/sl-7.x86_64/modules/cBlosc/2.0.4/include/' '-L/global/home/groups/software/sl-7.x86_64/modules/cBlosc/2.0.4/lib64' -lblosc2 -lz -luuid parallelreadzarr.cpp zarr.cpp helperfunctions.cpp parallelreadzarrwrapper.cpp + mex -v CXXOPTIMFLAGS="-DNDEBUG -O2" LDOPTIMFLAGS="-Wl',-rpath='''$ORIGIN'''' -O2 -DNDEBUG" CXXFLAGS='$CXXFLAGS -fopenmp -O2' LDFLAGS='$LDFLAGS -fopenmp -O2' -I'/clusterfs/fiona/matthewmueller/cppZarrTest' -I'/global/home/groups/software/sl-7.x86_64/modules/cBlosc/2.8.0/include/' '-L/global/home/groups/software/sl-7.x86_64/modules/cBlosc/2.8.0/lib64' -lblosc2 -lz -luuid parallelreadzarr.cpp zarr.cpp helperfunctions.cpp parallelreadzarrwrapper.cpp + + %mex -v CXXOPTIMFLAGS="-DNDEBUG -O3" LDOPTIMFLAGS="-Wl',-rpath='''$ORIGIN'''' -O3 -DNDEBUG" CXXFLAGS='$CXXFLAGS -fopenmp -O3' LDFLAGS='$LDFLAGS -fopenmp -O3' -I'/clusterfs/fiona/matthewmueller/cppZarrTest' -I'/global/home/groups/software/sl-7.x86_64/modules/cBlosc/2.8.0/include/' '-L/global/home/groups/software/sl-7.x86_64/modules/cBlosc/2.8.0/lib64' -lblosc2 -lz -luuid parallelreadzarr.cpp zarr.cpp helperfunctions.cpp parallelreadzarrwrapper.cpp + %mex -v CXXOPTIMFLAGS="-DNDEBUG -O2" LDOPTIMFLAGS="-Wl',-rpath='''$ORIGIN'''' -O2 -DNDEBUG" CXXFLAGS='$CXXFLAGS -fopenmp -O2' LDFLAGS='$LDFLAGS -fopenmp -O2' -I'/home/matt/c-zarr/cBlosc2/include' -L'/home/matt/c-zarr/cBlosc2/lib' -lblosc2 -lz -luuid parallelreadzarr.cpp zarr.cpp helperfunctions.cpp + end + % Need to change the library name because matlab preloads their own version + % of libstdc++ + % Setting it to libstdc++.so.6.0.30 as of MATLAB R2022b + %system('patchelf --replace-needed libc.so.6 libc.so.6.0.0 parallelreadzarr.mexa64'); + system('patchelf --replace-needed libstdc++.so.6 libstdc++.so.6.0.30 parallelreadzarr.mexa64'); + %system('export LD_LIBRARY_PATH="";patchelf --replace-needed libc.so.6 libc.so.6.0.0 parallelreadzarr.mexa64'); + %system('export LD_LIBRARY_PATH="";patchelf --replace-needed libstdc++.so.6 libstdc++.so.6.0.30 parallelreadzarr.mexa64'); + mkdir('../cpp-zarr_linux'); + movefile('parallelreadzarr.mexa64','../cpp-zarr_linux/parallelReadZarr.mexa64'); +elseif ismac + % Might have to do this part in terminal. First change the library + % linked to libstdc++ + % system('install_name_tool -change @rpath/libgcc_s.1.1.dylib @loader_path/libgcc_s.1.1.0.dylib ../cpp-zarr_mac/libstdc++.6.0.30.dylib'); + + + %mex -v CXX="/usr/local/bin/g++-12" CXXOPTIMFLAGS='-O2 -DNDEBUG' LDOPTIMFLAGS='-O2 -DNDEBUG' CXXFLAGS='$CXXFLAGS -O2 -fopenmp' LDFLAGS='$LDFLAGS -O2 -fopenmp' '-I/usr/local/include/' -L'/usr/local/lib/' -lstdc++ -lblosc2 -lz -luuid parallelreadzarr.cpp helperfunctions.cpp zarr.cpp + mex -v CXX="/usr/local/bin/g++-12" CXXOPTIMFLAGS='-O2 -DNDEBUG' LDOPTIMFLAGS='-O2 -DNDEBUG' CXXFLAGS='-fno-common -arch x86_64 -mmacosx-version-min=10.15 -fexceptions -isysroot /Library/Developer/CommandLineTools/SDKs/MacOSX.sdk -std=c++11 -O2 -fopenmp -DMATLAB_DEFAULT_RELEASE=R2017b -DUSE_MEX_CMD -DMATLAB_MEX_FILE' LDFLAGS='$LDFLAGS -O2 -fopenmp' '-I/usr/local/include/' -lstdc++ -lblosc2 -lz -luuid parallelreadzarr.cpp helperfunctions.cpp zarr.cpp parallelreadzarrwrapper.cpp + + % We need to change all the current paths to be relative to the mex file + system('install_name_tool -change /usr/local/opt/gcc@12/lib/gcc/12/libstdc++.6.dylib @loader_path/libstdc++.6.0.30.dylib parallelreadzarr.mexmaci64'); + system('install_name_tool -change /usr/local/opt/ossp-uuid/lib/libuuid.16.dylib @loader_path/libuuid.16.22.0.dylib parallelreadzarr.mexmaci64'); + system('install_name_tool -change /usr/local/opt/gcc@12/lib/gcc/12/libgomp.1.dylib @loader_path/libgomp.1.dylib parallelreadzarr.mexmaci64'); + system('install_name_tool -change @rpath/libblosc2.2.dylib @loader_path/libblosc2.2.8.0.dylib parallelreadzarr.mexmaci64'); + system('install_name_tool -change /usr/lib/libz.1.dylib @loader_path/libz.1.2.13.dylib parallelreadzarr.mexmaci64'); + system('install_name_tool -change /usr/local/opt/gcc@12/lib/gcc/12/libgcc_s.1.1.dylib @loader_path/libgcc_s.1.1.0.dylib parallelreadzarr.mexmaci64'); + + + system('chmod 777 parallelreadzarr.mexmaci64'); + mkdir('../cpp-zarr_mac'); + movefile('parallelreadzarr.mexmaci64','../cpp-zarr_mac/parallelReadZarr.mexmaci64'); +elseif ispc + mex CXX="C:/mingw64/bin/g++" -v CXXOPTIMFLAGS="-DNDEBUG -O2" LDOPTIMFLAGS="-Wl',-rpath='''$ORIGIN'''' -O2 -DNDEBUG" CXXFLAGS='$CXXFLAGS -fopenmp -O2' LDFLAGS='$LDFLAGS -fopenmp -O2' -I'C:/Program Files (x86)/nlohmann_json/include' -I'C:/Program Files (x86)/blosc2/include' '-LC:/Program Files (x86)/blosc2/lib' -I'C:/Program Files (x86)/zlib/include' '-LC:/Program Files (x86)/zlib/lib' -lblosc2.dll -lzlib.dll parallelreadzarr.cpp helperfunctions.cpp zarr.cpp parallelreadzarrwrapper.cpp + + mkdir('../cpp-zarr_windows'); + movefile('parallelreadzarr.mexw64','../cpp-zarr_windows/parallelReadZarr.mexw64'); +end diff --git a/mexSrc/compile_parallelWriteZarr.m b/mexSrc/compile_parallelWriteZarr.m new file mode 100644 index 0000000..6818249 --- /dev/null +++ b/mexSrc/compile_parallelWriteZarr.m @@ -0,0 +1,49 @@ +debug = false; +% ADD --disable-new-dtags +if isunix && ~ismac + if debug + mex -g -v LDOPTIMFLAGS="-Wl',-rpath='''$ORIGIN'''' -O0 -g" CXXFLAGS='$CXXFLAGS -fopenmp -O0 -g' LDFLAGS='$LDFLAGS -fopenmp -O0 -g' -I'/clusterfs/fiona/matthewmueller/cppZarrTest' -I'/global/home/groups/software/sl-7.x86_64/modules/cBlosc/2.8.0/include/' '-L/global/home/groups/software/sl-7.x86_64/modules/cBlosc/2.8.0/lib64' -I'/global/home/groups/software/sl-7.x86_64/modules/cBlosc/zarr/include/' '-L/global/home/groups/software/sl-7.x86_64/modules/cBlosc/zarr/lib' -lblosc -lblosc2 -lz -luuid parallelwritezarr.cpp helperfunctions.cpp zarr.cpp parallelreadzarrwrapper.cpp + %mex -v LDOPTIMFLAGS="-g -O0 -Wall -Wextra -Wl',-rpath='''$ORIGIN''''" CXXFLAGS='$CXXFLAGS -Wall -Wextra -g -O0 -fopenmp' LDFLAGS='$LDFLAGS -g -O0 -fopenmp' '-I/home/matt/c-zarr/cBlosc/include' '-I/home/matt/c-zarr/cBlosc2/include' '-L/home/matt/c-zarr/cBlosc/lib' -lblosc '-L/home/matt/c-zarr/cBlosc2/lib' -lblosc2 -lz -luuid parallelwritezarr.cpp helperfunctions.cpp zarr.cpp parallelreadzarrwrapper.cpp + else + %mex -v CXXOPTIMFLAGS="-DNDEBUG -O2" LDOPTIMFLAGS="-Wl',-rpath='''$ORIGIN'''' -O2 -DNDEBUG" CXXFLAGS='$CXXFLAGS -fopenmp -O2' LDFLAGS='$LDFLAGS -fopenmp -O2' '-I/home/matt/c-zarr/cBlosc/include' '-I/home/matt/c-zarr/cBlosc2/include' '-L/home/matt/c-zarr/cBlosc/lib' -lblosc '-L/home/matt/c-zarr/cBlosc2/lib' -lblosc2 -lz -luuid parallelwritezarr.cpp helperfunctions.cpp zarr.cpp parallelwritezarrread.cpp + mex -v CXXOPTIMFLAGS="-DNDEBUG -O2" LDOPTIMFLAGS="-Wl',-rpath='''$ORIGIN'''' -O2 -DNDEBUG" CXXFLAGS='$CXXFLAGS -fopenmp -O2' LDFLAGS='$LDFLAGS -fopenmp -O2' -I'/clusterfs/fiona/matthewmueller/cppZarrTest' -I'/global/home/groups/software/sl-7.x86_64/modules/cBlosc/2.8.0/include/' '-L/global/home/groups/software/sl-7.x86_64/modules/cBlosc/2.8.0/lib64' -I'/global/home/groups/software/sl-7.x86_64/modules/cBlosc/zarr/include/' '-L/global/home/groups/software/sl-7.x86_64/modules/cBlosc/zarr/lib' -lblosc -lblosc2 -lz -luuid parallelwritezarr.cpp helperfunctions.cpp zarr.cpp parallelreadzarrwrapper.cpp + end + % Need to change the library name because matlab preloads their own version + % of libstdc++ + % Setting it to libstdc++.so.6.0.30 as of MATLAB R2022b + system('patchelf --replace-needed libstdc++.so.6 libstdc++.so.6.0.30 parallelwritezarr.mexa64'); + %system('export LD_LIBRARY_PATH="";patchelf --replace-needed libstdc++.so.6 libstdc++.so.6.0.30 parallelWriteZarr.mexa64'); + mkdir('../cpp-zarr_linux'); + movefile('parallelwritezarr.mexa64','../cpp-zarr_linux/parallelWriteZarr.mexa64'); +elseif ismac + % Might have to do this part in terminal. First change the library + % linked to libstdc++ + % system('install_name_tool -change @rpath/libgcc_s.1.1.dylib @loader_path/libgcc_s.1.1.0.dylib ../cpp-zarr_mac/libstdc++.6.0.30.dylib'); + + + if debug + %mex -v -g CXX="/usr/local/opt/llvm/bin/clang++" CXXOPTIMFLAGS="" LDOPTIMFLAGS='-g -O0 -Wall -Wextra' CXXFLAGS='$CXXFLAGS -g -O0 -Wall -Wextra -fopenmp' LDFLAGS='$LDFLAGS -g -O0 -fopenmp' '-I/usr/local/include/' -L'/usr/local/lib/' -lblosc -lblosc2 -lz -luuid parallelwritezarr.cpp helperfunctions.cpp zarr.cpp parallelwritezarrread.cpp + %mex -v -g CXXOPTIMFLAGS='' LDOPTIMFLAGS='-g -O0' CXXFLAGS='$CXXFLAGS -O0 -Xpreprocessor -fopenmp' LDFLAGS='$LDFLAGS -g -O0 -Xpreprocessor -fopenmp' '-I/usr/local/opt/libomp/include' '-I/usr/include' '-I/usr/local/include/' -L'/usr/local/lib/' -L'/usr/local/opt/libomp/lib' -lomp -lblosc -lblosc2 -lz -luuid parallelwritezarr.cpp helperfunctions.cpp zarr.cpp parallelwritezarrread.cpp + else + mex -v CXX="/usr/local/bin/g++-12" CXXOPTIMFLAGS='-O2 -DNDEBUG' LDOPTIMFLAGS='-O2 -DNDEBUG' CXXFLAGS='-fno-common -arch x86_64 -mmacosx-version-min=10.15 -fexceptions -isysroot /Library/Developer/CommandLineTools/SDKs/MacOSX.sdk -std=c++11 -O2 -fopenmp -DMATLAB_DEFAULT_RELEASE=R2017b -DUSE_MEX_CMD -DMATLAB_MEX_FILE' LDFLAGS='$LDFLAGS -O2 -fopenmp' '-I/usr/local/include/' -lstdc++ -lblosc -lblosc2 -lz -luuid parallelwritezarr.cpp helperfunctions.cpp zarr.cpp parallelreadzarrwrapper.cpp + end + + % We need to change all the current paths to be relative to the mex file + system('install_name_tool -change /usr/local/opt/gcc@12/lib/gcc/12/libstdc++.6.dylib @loader_path/libstdc++.6.0.30.dylib parallelwritezarr.mexmaci64'); + system('install_name_tool -change /usr/local/opt/ossp-uuid/lib/libuuid.16.dylib @loader_path/libuuid.16.22.0.dylib parallelwritezarr.mexmaci64'); + system('install_name_tool -change /usr/local/opt/gcc@12/lib/gcc/12/libgomp.1.dylib @loader_path/libgomp.1.dylib parallelwritezarr.mexmaci64'); + system('install_name_tool -change @rpath/libblosc.1.dylib @loader_path/libblosc.1.21.0.0.dylib parallelwritezarr.mexmaci64'); + system('install_name_tool -change @rpath/libblosc2.2.dylib @loader_path/libblosc2.2.8.0.dylib parallelwritezarr.mexmaci64'); + system('install_name_tool -change /usr/lib/libz.1.dylib @loader_path/libz.1.2.13.dylib parallelwritezarr.mexmaci64'); + system('install_name_tool -change /usr/local/opt/gcc@12/lib/gcc/12/libgcc_s.1.1.dylib @loader_path/libgcc_s.1.1.0.dylib parallelwritezarr.mexmaci64'); + + + system('chmod 777 parallelwritezarr.mexmaci64'); + mkdir('../cpp-zarr_mac'); + movefile('parallelwritezarr.mexmaci64','../cpp-zarr_mac/parallelWriteZarr.mexmaci64'); +elseif ispc + mex CXX="C:/mingw64/bin/g++" -v CXXOPTIMFLAGS="-DNDEBUG -O2" LDOPTIMFLAGS="-Wl',-rpath='''$ORIGIN'''' -O2 -DNDEBUG" CXXFLAGS='$CXXFLAGS -fopenmp -O2' LDFLAGS='$LDFLAGS -fopenmp -O2' -I'C:/Program Files (x86)/nlohmann_json/include' -I'C:/Program Files (x86)/blosc2/include' '-LC:/Program Files (x86)/blosc2/lib' -I'C:/Program Files (x86)/blosc/include' '-LC:/Program Files (x86)/blosc/lib' -I'C:/Program Files (x86)/zlib/include' '-LC:/Program Files (x86)/zlib/lib' -lblosc.dll -lblosc2.dll -lzlib.dll parallelwritezarr.cpp helperfunctions.cpp zarr.cpp parallelreadzarrwrapper.cpp + + mkdir('../cpp-zarr_windows'); + movefile('parallelwritezarr.mexw64','../cpp-zarr_windows/parallelWriteZarr.mexw64'); +end \ No newline at end of file diff --git a/mexSrc/createzarrfile.cpp b/mexSrc/createzarrfile.cpp new file mode 100755 index 0000000..9f125ac --- /dev/null +++ b/mexSrc/createzarrfile.cpp @@ -0,0 +1,105 @@ +#include +#include +#include +#include +#include +#include +#include +#ifdef _WIN32 +#include +#include +#else +#include +#endif +#include +#include "mex.h" +#include "zarr.h" +#include "helperfunctions.h" + +void mexFunction(int nlhs, mxArray *plhs[], + int nrhs, const mxArray *prhs[]) +{ + zarr Zarr; + + if(nrhs < 1) mexErrMsgIdAndTxt("zarr:inputError","This functions requires at least 1 argument\n"); + if(nrhs == 2) mexErrMsgIdAndTxt("zarr:inputError","This functions does not accept only 2 arguments\n"); + if(!mxIsChar(prhs[0])) mexErrMsgIdAndTxt("zarr:inputError","The first argument must be a string\n"); + Zarr.set_fileName(mxArrayToString(prhs[0])); + + for(int i = 1; i < nrhs; i+=2){ + if(i+1 == nrhs) mexErrMsgIdAndTxt("zarr:inputError","Mismatched argument pair for input number %d\n"); + if(!mxIsChar(prhs[0])) mexErrMsgIdAndTxt("zarr:inputError","The argument in input location %d is not a string\n",i+1); + std::string currInput = mxArrayToString(prhs[i]); + + if(currInput == "chunks"){ + if(mxGetN(prhs[i+1]) != 3) mexErrMsgIdAndTxt("zarr:inputError","chunks must be an array of 3 numbers\n"); + Zarr.set_chunks({(uint64_t)*(mxGetPr(prhs[i+1])), + (uint64_t)*((mxGetPr(prhs[i+1])+1)), + (uint64_t)*((mxGetPr(prhs[i+1])+2))}); + } + else if(currInput == "cname"){ + if(!mxIsChar(prhs[i+1])) mexErrMsgIdAndTxt("zarr:inputError","cname must be a string\n"); + Zarr.set_cname(mxArrayToString(prhs[i+1])); + } + else if(currInput == "dtype"){ + if(!mxIsChar(prhs[i+1])) mexErrMsgIdAndTxt("zarr:inputError","dtype must be a string\n"); + std::string dtype = mxArrayToString(prhs[i+1]); + if(dtype.size() == 2){ + dtype.insert(0,1,'<'); + } + else if(dtype.size() != 3) mexErrMsgIdAndTxt("zarr:inputError","dtype must be of length 2 or 3\n"); + + if(dtype[0] != '<' && dtype[0] != '>'){ + mexErrMsgIdAndTxt("zarr:inputError","The first character of dtype must be \"<\" or \">\"\n"); + } + + if(dtype[1] == 'u' || dtype[1] == 'f'){ + if(dtype[1] == 'u'){ + if(dtype[2] != '1' && dtype[2] != '2'){ + mexErrMsgIdAndTxt("zarr:inputError","For dtype u, the third character of dtype must be \"1\" or \"2\"\n"); + } + } + else{ + if(dtype[2] != '4' && dtype[2] != '8'){ + mexErrMsgIdAndTxt("zarr:inputError","For dtype f, the third character of dtype must be \"4\" or \"8\"\n"); + } + } + } + else mexErrMsgIdAndTxt("zarr:inputError","The second character of dtype must be \"u\" or \"f\"\n"); + Zarr.set_dtype(dtype); + } + else if(currInput == "order"){ + if(!mxIsChar(prhs[i+1])) mexErrMsgIdAndTxt("zarr:inputError","order must be a string\n"); + std::string order = mxArrayToString(prhs[i+1]); + if(order.size() > 1) mexErrMsgIdAndTxt("zarr:inputError","order must be of length 1\n"); + if(order[0] != 'F' || order[0] != 'C' || order[0] != 'f' || order[0] != 'c'){ + if(order[0] == 'f') order = "F"; + else if(order [0] == 'c') order = "C"; + } + else mexErrMsgIdAndTxt("zarr:inputError","order must be \"F\" or \"C\"\n"); + Zarr.set_order(order); + } + else if(currInput == "shape"){ + if(mxGetN(prhs[i+1]) != 3) mexErrMsgIdAndTxt("zarr:inputError","shape must be an array of 3 numbers\n"); + Zarr.set_shape({(uint64_t)*(mxGetPr(prhs[i+1])), + (uint64_t)*((mxGetPr(prhs[i+1])+1)), + (uint64_t)*((mxGetPr(prhs[i+1])+2))}); + } + else if(currInput == "clevel"){ + if(!mxIsNumeric(prhs[i+1])) mexErrMsgIdAndTxt("zarr:inputError","clevel must be a numerical value\n"); + Zarr.set_clevel((uint64_t)*(mxGetPr(prhs[i+1]))); + } + else if(currInput == "subfolders"){ + if(mxGetN(prhs[i+1]) != 3) mexErrMsgIdAndTxt("zarr:inputError","subfolders must be an array of 3 numbers\n"); + Zarr.set_subfolders({(uint64_t)*(mxGetPr(prhs[i+1])), + (uint64_t)*((mxGetPr(prhs[i+1])+1)), + (uint64_t)*((mxGetPr(prhs[i+1])+2))}); + } + else{ + mexErrMsgIdAndTxt("zarr:inputError","The argument \"%s\" does not match the name of any supported input name.\n \ + Currently Supported Names: chunks, cname, dtype, order, shape\n",currInput.c_str()); + } + } + + Zarr.write_zarray(); +} \ No newline at end of file diff --git a/mexSrc/helperfunctions.cpp b/mexSrc/helperfunctions.cpp new file mode 100644 index 0000000..eeb4c13 --- /dev/null +++ b/mexSrc/helperfunctions.cpp @@ -0,0 +1,131 @@ +#include +#include +#include +#include +#include +#include +#ifdef _WIN32 +#include +#include +#else +#include +#endif +#include +#include "mex.h" +#include "helperfunctions.h" + +#ifndef _WIN32 +#include +// Expand the tilde to the home directory on Mac and Linux +const char* expandTilde(const char* path) { + if(strchr(path,'~')){ + wordexp_t expPath; + wordexp(path, &expPath, 0); + return expPath.we_wordv[0]; + } + else return path; +} +#endif + +#ifdef _WIN32 +char* strndup (const char *s, size_t n) +{ + size_t len = strnlen (s, n); + char *newS = (char *) malloc (len + 1); + if (newS == NULL) + return NULL; + newS[len] = '\0'; + return (char *) memcpy (newS, s, len); +} +int _vscprintf_so(const char * format, va_list pargs) { + int retval; + va_list argcopy; + va_copy(argcopy, pargs); + retval = vsnprintf(NULL, 0, format, argcopy); + va_end(argcopy); + return retval; +} + +int vasprintf(char **strp, const char *fmt, va_list ap) { + int len = _vscprintf_so(fmt, ap); + if (len == -1) return -1; + char *str = (char*)malloc((size_t) len + 1); + if (!str) return -1; + int r = vsnprintf(str, len + 1, fmt, ap); /* "secure" version of vsprintf */ + if (r == -1) return free(str), -1; + *strp = str; + return r; +} + +int asprintf(char *strp[], const char *fmt, ...) { + va_list ap; + va_start(ap, fmt); + int r = vasprintf(strp, fmt, ap); + va_end(ap); + return r; +} +#endif + +std::string generateUUID(){ + #ifdef _WIN32 + // uuid of length 5 for windows + char uuidC[6]; + char *seedArr = (char*)malloc(1000); + struct timeval cSeed; + gettimeofday(&cSeed,NULL); + int nChars = sprintf(seedArr,"%d%d",cSeed.tv_sec,cSeed.tv_usec); + int aSeed = 0; + char* ptr; + if(nChars > 9) + aSeed = strtol(seedArr+nChars-10, &ptr, 9); + else aSeed = strtol(seedArr, &ptr, 9); + srand(aSeed); + sprintf(uuidC,"%.5d",rand() % 99999); + free(seedArr); + #else + uuid_t binuuid; + uuid_generate_random(binuuid); + char uuidC[37]; + uuid_unparse(binuuid, uuidC); + #endif + return std::string(uuidC); +} + +// Recursively make a directory and set its permissions to 775 +void mkdirRecursive(const char *dir) { + char tmp[8192]; + char *p = NULL; + size_t len; + snprintf(tmp, sizeof(tmp),"%s",dir); + len = strlen(tmp); + if (tmp[len - 1] == '/') + tmp[len - 1] = 0; + for (p = tmp + 1; *p; p++){ + if (*p == '/') { + *p = 0; + + #ifdef _WIN32 + mkdir(tmp); + #else + mkdir(tmp, 0775); + #endif + + chmod(tmp, 0775); + *p = '/'; + } + } + #ifdef _WIN32 + mkdir(tmp); + #else + mkdir(tmp, 0775); + #endif + chmod(tmp, 0775); +} + +bool fileExists(const std::string &fileName){ + if (FILE *file = fopen(fileName.c_str(), "r")){ + fclose(file); + return true; + } + else return false; +} \ No newline at end of file diff --git a/mexSrc/helperfunctions.h b/mexSrc/helperfunctions.h new file mode 100644 index 0000000..e84a249 --- /dev/null +++ b/mexSrc/helperfunctions.h @@ -0,0 +1,24 @@ +#ifndef HELPERFUNCTIONS_H +#define HELPERFUNCTIONS_H +#include + +#ifndef _WIN32 +const char* expandTilde(const char* path); +#endif + +#ifdef _WIN32 +char* strndup (const char *s, size_t n); + +int _vscprintf_so(const char * format, va_list pargs); + +int vasprintf(char **strp, const char *fmt, va_list ap); + +int asprintf(char *strp[], const char *fmt, ...); +#endif + +std::string generateUUID(); + +void mkdirRecursive(const char *dir); + +bool fileExists(const std::string &fileName); +#endif \ No newline at end of file diff --git a/mexSrc/parallelreadzarr.cpp b/mexSrc/parallelreadzarr.cpp new file mode 100644 index 0000000..0dbb92a --- /dev/null +++ b/mexSrc/parallelreadzarr.cpp @@ -0,0 +1,83 @@ +#include +#include +#include "parallelreadzarrwrapper.h" +#include "blosc2.h" +#include "mex.h" +#include "zarr.h" +#include "helperfunctions.h" +#include "zlib.h" + +// TODO: FIX MEMORY LEAKS +void mexFunction(int nlhs, mxArray *plhs[], + int nrhs, const mxArray *prhs[]) +{ + std::vector startCoords = {0,0,0}; + std::vector endCoords = {0,0,0}; + + if(!nrhs) mexErrMsgIdAndTxt("zarr:inputError","This functions requires at least 1 argument"); + else if(nrhs == 2){ + if(mxGetN(prhs[1]) != 6) mexErrMsgIdAndTxt("zarr:inputError","Input range is not 6"); + startCoords[0] = (uint64_t)*(mxGetPr(prhs[1]))-1; + startCoords[1] = (uint64_t)*((mxGetPr(prhs[1])+1))-1; + startCoords[2] = (uint64_t)*((mxGetPr(prhs[1])+2))-1; + endCoords[0] = (uint64_t)*((mxGetPr(prhs[1])+3)); + endCoords[1] = (uint64_t)*((mxGetPr(prhs[1])+4)); + endCoords[2] = (uint64_t)*((mxGetPr(prhs[1])+5)); + + if(startCoords[0]+1 < 1 || startCoords[1]+1 < 1 || startCoords[2]+1 < 1) mexErrMsgIdAndTxt("zarr:inputError","Lower bounds must be at least 1"); + } + else if (nrhs > 2) mexErrMsgIdAndTxt("zarr:inputError","Number of input arguments must be 1 or 2"); + if(!mxIsChar(prhs[0])) mexErrMsgIdAndTxt("zarr:inputError","The first argument must be a string"); + std::string folderName(mxArrayToString(prhs[0])); + + // Handle the tilde character in filenames on Linux/Mac + #ifndef _WIN32 + folderName = expandTilde(folderName.c_str()); + #endif + + zarr Zarr(folderName); + + + if(endCoords[0] > Zarr.get_shape(0) || + endCoords[1] > Zarr.get_shape(1) || + endCoords[2] > Zarr.get_shape(2)) mexErrMsgIdAndTxt("zarr:inputError","Upper bound is invalid"); + if(nrhs == 1){ + endCoords[0] = Zarr.get_shape(0); + endCoords[1] = Zarr.get_shape(1); + endCoords[2] = Zarr.get_shape(2); + } + const std::vector readShape = {endCoords[0]-startCoords[0], + endCoords[1]-startCoords[1], + endCoords[2]-startCoords[2]}; + uint64_t dim[3] = {readShape[0],readShape[1],readShape[2]}; + + Zarr.set_chunkInfo(startCoords, endCoords); + if(Zarr.get_dtype().find("u1") != std::string::npos){ + uint64_t bits = 8; + plhs[0] = mxCreateNumericArray(3,(mwSize*)dim,mxUINT8_CLASS, mxREAL); + uint8_t* zarrArr = (uint8_t*)mxGetPr(plhs[0]); + parallelReadZarrMex(Zarr, (void*)zarrArr,startCoords,endCoords,readShape,bits); + } + else if(Zarr.get_dtype().find("u2") != std::string::npos){ + uint64_t bits = 16; + plhs[0] = mxCreateNumericArray(3,(mwSize*)dim,mxUINT16_CLASS, mxREAL); + uint16_t* zarrArr = (uint16_t*)mxGetPr(plhs[0]); + parallelReadZarrMex(Zarr, (void*)zarrArr,startCoords,endCoords,readShape,bits); + } + else if(Zarr.get_dtype().find("f4") != std::string::npos){ + uint64_t bits = 32; + plhs[0] = mxCreateNumericArray(3,(mwSize*)dim,mxSINGLE_CLASS, mxREAL); + float* zarrArr = (float*)mxGetPr(plhs[0]); + parallelReadZarrMex(Zarr, (void*)zarrArr,startCoords,endCoords,readShape,bits); + } + else if(Zarr.get_dtype().find("f8") != std::string::npos){ + uint64_t bits = 64; + plhs[0] = mxCreateNumericArray(3,(mwSize*)dim,mxDOUBLE_CLASS, mxREAL); + double* zarrArr = (double*)mxGetPr(plhs[0]); + parallelReadZarrMex(Zarr, (void*)zarrArr,startCoords,endCoords,readShape,bits); + } + else{ + mexErrMsgIdAndTxt("tiff:dataTypeError","Data type not suppported"); + } + +} diff --git a/mexSrc/parallelreadzarrwrapper.cpp b/mexSrc/parallelreadzarrwrapper.cpp new file mode 100644 index 0000000..ccb06c7 --- /dev/null +++ b/mexSrc/parallelreadzarrwrapper.cpp @@ -0,0 +1,292 @@ +#include +#include +#include +#include "parallelreadzarrwrapper.h" +#include "blosc2.h" +#include "mex.h" +#include "zarr.h" +#include "helperfunctions.h" +#include "zlib.h" + +void parallelReadZarrMex(const zarr &Zarr, void* zarrArr, + const std::vector &startCoords, + const std::vector &endCoords, + const std::vector &readShape, + const uint64_t bits, + const bool useCtx) +{ + const uint64_t bytes = (bits/8); + + int32_t numWorkers = omp_get_max_threads(); + + // nBloscThreads used when using blosc_ctx + uint32_t nBloscThreads = 1; + if(!useCtx){ + blosc2_init(); + blosc2_set_nthreads(numWorkers); + } + + // no ctx + /* + if(numWorkers>Zarr.get_numChunks()){ + blosc_set_nthreads(std::ceil(((double)numWorkers)/((double)Zarr.get_numChunks()))); + numWorkers = Zarr.get_numChunks(); + } + else { + blosc_set_nthreads(numWorkers); + } + */ + + // ctx + /* + if(numWorkers>Zarr.get_numChunks()){ + nBloscThreads = std::ceil(((double)numWorkers)/((double)Zarr.get_numChunks())); + numWorkers = Zarr.get_numChunks(); + } + */ + + const int32_t batchSize = (Zarr.get_numChunks()-1)/numWorkers+1; + const uint64_t s = Zarr.get_chunks(0)*Zarr.get_chunks(1)*Zarr.get_chunks(2); + const uint64_t sB = s*bytes; + + int err = 0; + std::string errString; + + #pragma omp parallel for + for(int32_t w = 0; w < numWorkers; w++){ + //void* bufferDest = malloc(sB); + //void* buffer = malloc(sB); + void* bufferDest = operator new(sB); + std::streamsize lastFileLen = 0; + void* buffer = NULL; + int64_t dsize = -1; + int uncErr = 0; + for(int64_t f = w*batchSize; f < (w+1)*batchSize; f++){ + if(f>=Zarr.get_numChunks() || err) break; + const std::vector cAV = Zarr.get_chunkAxisVals(Zarr.get_chunkNames(f)); + const std::string subfolderName = Zarr.get_subfoldersString(cAV); + + const std::string fileName(Zarr.get_fileName()+"/"+subfolderName+"/"+Zarr.get_chunkNames(f)); + + // If we cannot open the file then set to all zeros + // Can make this better by checking the errno + std::ifstream file(fileName, std::ios::binary); + if(!file.is_open()){ + memset(bufferDest,0,sB); + } + else{ + file.seekg(0, std::ios::end); + const std::streamsize fileLen = file.tellg(); + if(lastFileLen < fileLen){ + operator delete(buffer); + buffer = operator new(fileLen); + lastFileLen = fileLen; + } + file.seekg(0, std::ios::beg); + file.read(reinterpret_cast(buffer), fileLen); + file.close(); + + // Decompress + if(Zarr.get_cname() != "gzip"){ + if(!useCtx){ + dsize = blosc2_decompress(buffer, fileLen, bufferDest, sB); + } + else{ + blosc2_context *dctx; + blosc2_dparams dparams = {(int16_t)nBloscThreads,NULL,NULL,NULL}; + dctx = blosc2_create_dctx(dparams); + dsize = blosc2_decompress_ctx(dctx, buffer, fileLen, bufferDest, sB); + blosc2_free_ctx(dctx); + } + } + else{ + dsize = sB; + z_stream stream; + stream.zalloc = Z_NULL; + stream.zfree = Z_NULL; + stream.opaque = Z_NULL; + stream.avail_in = (uInt)fileLen; + stream.avail_out = (uInt)dsize; + while(stream.avail_in > 0){ + + dsize = sB; + + stream.next_in = (uint8_t*)buffer+(fileLen-stream.avail_in); + stream.next_out = (uint8_t*)bufferDest+(sB-stream.avail_out); + + uncErr = inflateInit2(&stream, 32); + if(uncErr){ + #pragma omp critical + { + err = 1; + errString = "Decompression error. Error code: "+ + std::to_string(uncErr)+" ChunkName: "+ + Zarr.get_fileName()+"/"+subfolderName+"/"+ + Zarr.get_chunkNames(f)+"\n"; + } + break; + } + + uncErr = inflate(&stream, Z_NO_FLUSH); + + if(uncErr != Z_STREAM_END){ + #pragma omp critical + { + err = 1; + errString = "Decompression error. Error code: "+ + std::to_string(uncErr)+" ChunkName: "+ + Zarr.get_fileName()+"/"+subfolderName+"/"+ + Zarr.get_chunkNames(f)+"\n"; + } + break; + } + } + if(inflateEnd(&stream)){ + #pragma omp critical + { + err = 1; + errString = "Decompression error. Error code: "+ + std::to_string(uncErr)+" ChunkName: "+ + Zarr.get_fileName()+"/"+subfolderName+"/"+ + Zarr.get_chunkNames(f)+"\n"; + } + break; + } + } + + + if(dsize < 0){ + #pragma omp critical + { + err = 1; + errString = "Decompression error. Error code: "+ + std::to_string(uncErr)+" ChunkName: "+ + Zarr.get_fileName()+"/"+subfolderName+"/"+ + Zarr.get_chunkNames(f)+"\n"; + } + break; + } + } + + if(Zarr.get_order() == "F"){ + for(int64_t z = cAV[2]*Zarr.get_chunks(2); z < (cAV[2]+1)*Zarr.get_chunks(2); z++){ + if(z>=endCoords[2]) break; + else if(z=endCoords[1]) break; + else if(y startCoords[0]) || (cAV[0]+1)*Zarr.get_chunks(0)>endCoords[0]){ + if(((cAV[0]*Zarr.get_chunks(0)) < startCoords[0] && ((cAV[0]+1)*Zarr.get_chunks(0)) > startCoords[0]) && (cAV[0]+1)*Zarr.get_chunks(0)>endCoords[0]){ + memcpy((uint8_t*)zarrArr+((((cAV[0]*Zarr.get_chunks(0))-startCoords[0]+(startCoords[0]%Zarr.get_chunks(0)))+((y-startCoords[1])*readShape[0])+((z-startCoords[2])*readShape[0]*readShape[1]))*bytes),(uint8_t*)bufferDest+(((startCoords[0]%Zarr.get_chunks(0))+((y%Zarr.get_chunks(1))*Zarr.get_chunks(0))+((z%Zarr.get_chunks(2))*Zarr.get_chunks(0)*Zarr.get_chunks(1)))*bytes),((endCoords[0]%Zarr.get_chunks(0))-(startCoords[0]%Zarr.get_chunks(0)))*bytes); + } + else if((cAV[0]+1)*Zarr.get_chunks(0)>endCoords[0]){ + memcpy((uint8_t*)zarrArr+((((cAV[0]*Zarr.get_chunks(0))-startCoords[0])+((y-startCoords[1])*readShape[0])+((z-startCoords[2])*readShape[0]*readShape[1]))*bytes),(uint8_t*)bufferDest+((((y%Zarr.get_chunks(1))*Zarr.get_chunks(0))+((z%Zarr.get_chunks(2))*Zarr.get_chunks(0)*Zarr.get_chunks(1)))*bytes),(endCoords[0]%Zarr.get_chunks(0))*bytes); + } + else if((cAV[0]*Zarr.get_chunks(0)) < startCoords[0] && ((cAV[0]+1)*Zarr.get_chunks(0)) > startCoords[0]){ + memcpy((uint8_t*)zarrArr+((((cAV[0]*Zarr.get_chunks(0)-startCoords[0]+(startCoords[0]%Zarr.get_chunks(0))))+((y-startCoords[1])*readShape[0])+((z-startCoords[2])*readShape[0]*readShape[1]))*bytes),(uint8_t*)bufferDest+(((startCoords[0]%Zarr.get_chunks(0))+((y%Zarr.get_chunks(1))*Zarr.get_chunks(0))+((z%Zarr.get_chunks(2))*Zarr.get_chunks(0)*Zarr.get_chunks(1)))*bytes),(Zarr.get_chunks(0)-(startCoords[0]%Zarr.get_chunks(0)))*bytes); + } + } + else{ + memcpy((uint8_t*)zarrArr+((((cAV[0]*Zarr.get_chunks(0))-startCoords[0])+((y-startCoords[1])*readShape[0])+((z-startCoords[2])*readShape[0]*readShape[1]))*bytes),(uint8_t*)bufferDest+((((y%Zarr.get_chunks(1))*Zarr.get_chunks(0))+((z%Zarr.get_chunks(2))*Zarr.get_chunks(0)*Zarr.get_chunks(1)))*bytes),Zarr.get_chunks(0)*bytes); + } + } + } + + } + else if (Zarr.get_order() == "C"){ + for(int64_t x = cAV[0]*Zarr.get_chunks(0); x < (cAV[0]+1)*Zarr.get_chunks(0); x++){ + if(x>=endCoords[0]) break; + else if(x=endCoords[1]) break; + else if(y=endCoords[2]) break; + else if(z startCoords, + std::vector endCoords){ + + if(!crop){ + startCoords[0] = 0; + startCoords[1] = 0; + startCoords[2] = 0; + endCoords[0] = Zarr.get_shape(0); + endCoords[1] = Zarr.get_shape(1); + endCoords[2] = Zarr.get_shape(2); + } + else{ + startCoords[0]--; + startCoords[1]--; + startCoords[2]--; + } + + + std::vector readShape = {endCoords[0]-startCoords[0], + endCoords[1]-startCoords[1], + endCoords[2]-startCoords[2]}; + + Zarr.set_chunkInfo(startCoords, endCoords); + if(Zarr.get_dtype() == " &startCoords, + const std::vector &endCoords, + const std::vector &readShape, + const uint64_t bits, + const bool useCtx=false); + +void* parallelReadZarrWrapper(zarr Zarr, const bool &crop, + std::vector startCoords, + std::vector endCoords); +#endif \ No newline at end of file diff --git a/mexSrc/parallelwritezarr.cpp b/mexSrc/parallelwritezarr.cpp new file mode 100644 index 0000000..75aeeba --- /dev/null +++ b/mexSrc/parallelwritezarr.cpp @@ -0,0 +1,619 @@ +#include +#include +#include +#include +#include +#include +#include +#ifdef _WIN32 +#include +#else +#include +#endif +#include +#include +#include +#include "blosc.h" +#include "mex.h" +#include "helperfunctions.h" +#include "zarr.h" +#include "parallelreadzarrwrapper.h" +#include "zlib.h" + +//compile +//mex -v COPTIMFLAGS="-DNDEBUG -O3" CFLAGS='$CFLAGS -fopenmp -O3' LDFLAGS='$LDFLAGS -fopenmp -O3' '-I/global/home/groups/software/sl-7.x86_64/modules/cBlosc/2.0.4/include/' '-I/global/home/groups/software/sl-7.x86_64/modules/cBlosc/zarr/include/' '-I/global/home/groups/software/sl-7.x86_64/modules/cJSON/1.7.15/include/' '-L/global/home/groups/software/sl-7.x86_64/modules/cBlosc/zarr/lib' -lblosc '-L/global/home/groups/software/sl-7.x86_64/modules/cBlosc/2.0.4/lib64' -lblosc2 '-L/global/home/groups/software/sl-7.x86_64/modules/cJSON/1.7.15/lib64' -lcjson -luuid parallelWriteZarr.c helperFunctions.c parallelReadZarr.c + +//With zlib +//mex -v COPTIMFLAGS="-DNDEBUG -O3" LDOPTIMFLAGS="-Wl',-rpath='''$ORIGIN'''' -O3 -DNDEBUG" CFLAGS='$CFLAGS -fopenmp -O3' LDFLAGS='$LDFLAGS -fopenmp -O3' '-I/global/home/groups/software/sl-7.x86_64/modules/cBlosc/2.0.4/include/' '-I/global/home/groups/software/sl-7.x86_64/modules/cBlosc/zarr/include/' '-I/global/home/groups/software/sl-7.x86_64/modules/cJSON/1.7.15/include/' '-L/global/home/groups/software/sl-7.x86_64/modules/cBlosc/zarr/lib' -lblosc '-L/global/home/groups/software/sl-7.x86_64/modules/cBlosc/2.0.4/lib64' -lblosc2 '-L/global/home/groups/software/sl-7.x86_64/modules/cJSON/1.7.15/lib64' -lcjson -luuid -lz parallelWriteZarr.c helperFunctions.c parallelReadZarr.c + +//mex -v COPTIMFLAGS="-O3 -fwrapv -DNDEBUG" CFLAGS='$CFLAGS -O3 -fopenmp' LDFLAGS='$LDFLAGS -O3 -fopenmp' '-I/global/home/groups/software/sl-7.x86_64/modules/cBlosc/2.0.4/include/' '-L/global/home/groups/software/sl-7.x86_64/modules/cBlosc/2.0.4/lib64' -lblosc2 zarrMex.c +// +//Windows +//mex -v COPTIMFLAGS="-O3 -DNDEBUG" CFLAGS='$CFLAGS -O3 -fopenmp' LDFLAGS='$LDFLAGS -O3 -fopenmp' '-IC:\Program Files (x86)\bloscZarr\include' '-LC:\Program Files (x86)\bloscZarr\lib' -lblosc '-IC:\Program Files (x86)\cJSON\include\' '-LC:\Program Files (x86)\cJSON\lib' -lcjson '-IC:\Program Files (x86)\blosc\include' '-LC:\Program Files (x86)\blosc\lib' -lblosc2 parallelWriteZarr.c parallelReadZarr.c helperFunctions.c + +void parallelWriteZarrMex(zarr &Zarr, void* zarrArr, + const std::vector &startCoords, + const std::vector &endCoords, + const std::vector &writeShape, + const uint64_t bits, const bool useUuid, const bool crop){ + //printf("%s startCoords[0]yz: %d %d %d endCoords[0]yz: %d %d %d chunkxyz: %d %d %d writeShape[0]yz: %d %d %d bits: %d\n",Zarr.get_fileName().c_str(),startCoords[0],startCoords[1],startCoords[2],endCoords[0],endCoords[1],endCoords[2],Zarr.get_chunks(0),Zarr.get_chunks(1),Zarr.get_chunks(2),writeShape[0],writeShape[1],writeShape[2],bits); + const uint64_t bytes = (bits/8); + + int32_t numWorkers = omp_get_max_threads(); + + int32_t nBloscThreads = 1; + if(numWorkers>Zarr.get_numChunks()){ + nBloscThreads = std::ceil(((double)numWorkers)/((double)Zarr.get_numChunks())); + numWorkers = Zarr.get_numChunks(); + } + + const int32_t batchSize = (Zarr.get_numChunks()-1)/numWorkers+1; + const uint64_t s = Zarr.get_chunks(0)*Zarr.get_chunks(1)*Zarr.get_chunks(2); + const uint64_t sB = s*bytes; + + const std::string uuid(generateUUID()); + + int err = 0; + std::string errString; + + #pragma omp parallel for + for(int32_t w = 0; w < numWorkers; w++){ + void* chunkUnC = malloc(sB); + void* chunkC = malloc(sB+BLOSC_MAX_OVERHEAD); + for(int64_t f = w*batchSize; f < (w+1)*batchSize; f++){ + if(f>=Zarr.get_numChunks() || err) break; + const std::vector cAV = Zarr.get_chunkAxisVals(Zarr.get_chunkNames(f)); + void* cRegion = nullptr; + + if(crop && ((((cAV[0])*Zarr.get_chunks(0)) < startCoords[0] || ((cAV[0]+1)*Zarr.get_chunks(0) > endCoords[0] && endCoords[0] < Zarr.get_shape(0))) + || (((cAV[1])*Zarr.get_chunks(1)) < startCoords[1] || ((cAV[1]+1)*Zarr.get_chunks(1) > endCoords[1] && endCoords[1] < Zarr.get_shape(1))) + || (((cAV[2])*Zarr.get_chunks(2)) < startCoords[2] || ((cAV[2]+1)*Zarr.get_chunks(2) > endCoords[2] && endCoords[2] < Zarr.get_shape(2))))){ + cRegion = parallelReadZarrWrapper(Zarr, crop, + {((cAV[0])*Zarr.get_chunks(0))+1, + ((cAV[1])*Zarr.get_chunks(1))+1, + ((cAV[2])*Zarr.get_chunks(2))+1}, + {(cAV[0]+1)*Zarr.get_chunks(0), + (cAV[1]+1)*Zarr.get_chunks(1), + (cAV[2]+1)*Zarr.get_chunks(2)}); + } + if(Zarr.get_order() == "F"){ + for(int64_t z = cAV[2]*Zarr.get_chunks(2); z < (cAV[2]+1)*Zarr.get_chunks(2); z++){ + if(z>=endCoords[2]){ + if(crop){ + if((cAV[2]+1)*Zarr.get_chunks(2) > Zarr.get_shape(2)){ + memcpy((uint8_t*)chunkUnC+((((z%Zarr.get_chunks(2))*Zarr.get_chunks(0)*Zarr.get_chunks(1)))*bytes),(uint8_t*)cRegion+((((z%Zarr.get_chunks(2))*Zarr.get_chunks(0)*Zarr.get_chunks(1)))*bytes),((Zarr.get_shape(2)-z)*Zarr.get_chunks(0)*Zarr.get_chunks(1))*bytes); + uint64_t zRest = ((cAV[2]+1)*Zarr.get_chunks(2))-Zarr.get_shape(2); + memset((uint8_t*)chunkUnC+(((z%Zarr.get_chunks(2))*Zarr.get_chunks(0)*Zarr.get_chunks(1))*bytes),0,(zRest*(Zarr.get_chunks(0)*Zarr.get_chunks(1)))*bytes); + } + else{ + memcpy((uint8_t*)chunkUnC+((((z%Zarr.get_chunks(2))*Zarr.get_chunks(0)*Zarr.get_chunks(1)))*bytes),(uint8_t*)cRegion+((((z%Zarr.get_chunks(2))*Zarr.get_chunks(0)*Zarr.get_chunks(1)))*bytes),((((cAV[2]+1)*Zarr.get_chunks(2))-z)*Zarr.get_chunks(0)*Zarr.get_chunks(1))*bytes); + } + } + else{ + uint64_t zRest = ((cAV[2]+1)*Zarr.get_chunks(2))-z; + memset((uint8_t*)chunkUnC+(((z%Zarr.get_chunks(2))*Zarr.get_chunks(0)*Zarr.get_chunks(1))*bytes),0,(zRest*(Zarr.get_chunks(0)*Zarr.get_chunks(1)))*bytes); + } + break; + } + else if(z=endCoords[1]){ + if(crop){ + if((cAV[1]+1)*Zarr.get_chunks(1) > Zarr.get_shape(1)){ + memcpy((uint8_t*)chunkUnC+((((y%Zarr.get_chunks(1))*Zarr.get_chunks(0))+((z%Zarr.get_chunks(2))*Zarr.get_chunks(0)*Zarr.get_chunks(1)))*bytes),(uint8_t*)cRegion+((((y%Zarr.get_chunks(1))*Zarr.get_chunks(0))+((z%Zarr.get_chunks(2))*Zarr.get_chunks(0)*Zarr.get_chunks(1)))*bytes),((Zarr.get_shape(1)-y)*Zarr.get_chunks(0))*bytes); + uint64_t yRest = ((cAV[1]+1)*Zarr.get_chunks(1))-Zarr.get_shape(1); + memset((uint8_t*)chunkUnC+((((y%Zarr.get_chunks(1))*Zarr.get_chunks(0))+((z%Zarr.get_chunks(2))*Zarr.get_chunks(0)*Zarr.get_chunks(1)))*bytes),0,(yRest*(Zarr.get_chunks(0)))*bytes); + } + else{ + memcpy((uint8_t*)chunkUnC+((((y%Zarr.get_chunks(1))*Zarr.get_chunks(0))+((z%Zarr.get_chunks(2))*Zarr.get_chunks(0)*Zarr.get_chunks(1)))*bytes),(uint8_t*)cRegion+((((y%Zarr.get_chunks(1))*Zarr.get_chunks(0))+((z%Zarr.get_chunks(2))*Zarr.get_chunks(0)*Zarr.get_chunks(1)))*bytes),((((cAV[1]+1)*Zarr.get_chunks(1))-y)*Zarr.get_chunks(0))*bytes); + } + } + else{ + uint64_t yRest = ((cAV[1]+1)*Zarr.get_chunks(1))-y; + memset((uint8_t*)chunkUnC+((((y%Zarr.get_chunks(1))*Zarr.get_chunks(0))+((z%Zarr.get_chunks(2))*Zarr.get_chunks(0)*Zarr.get_chunks(1)))*bytes),0,(yRest*Zarr.get_chunks(0))*bytes); + } + break; + } + else if(y startCoords[0]) || (cAV[0]+1)*Zarr.get_chunks(0)>endCoords[0]){ + if(((cAV[0]*Zarr.get_chunks(0)) < startCoords[0] && ((cAV[0]+1)*Zarr.get_chunks(0)) > startCoords[0]) && (cAV[0]+1)*Zarr.get_chunks(0)>endCoords[0]){ + if(crop){ + memcpy((uint8_t*)chunkUnC+((((y%Zarr.get_chunks(1))*Zarr.get_chunks(0))+((z%Zarr.get_chunks(2))*Zarr.get_chunks(0)*Zarr.get_chunks(1)))*bytes),(uint8_t*)cRegion+((((y%Zarr.get_chunks(1))*Zarr.get_chunks(0))+((z%Zarr.get_chunks(2))*Zarr.get_chunks(0)*Zarr.get_chunks(1)))*bytes),(startCoords[0]%Zarr.get_chunks(0))*bytes); + memcpy((uint8_t*)chunkUnC+(((startCoords[0]%Zarr.get_chunks(0))+((y%Zarr.get_chunks(1))*Zarr.get_chunks(0))+((z%Zarr.get_chunks(2))*Zarr.get_chunks(0)*Zarr.get_chunks(1)))*bytes),(uint8_t*)zarrArr+((((cAV[0]*Zarr.get_chunks(0))-startCoords[0]+(startCoords[0]%Zarr.get_chunks(0)))+((y-startCoords[1])*writeShape[0])+((z-startCoords[2])*writeShape[0]*writeShape[1]))*bytes),((endCoords[0]%Zarr.get_chunks(0))-(startCoords[0]%Zarr.get_chunks(0)))*bytes); + memcpy((uint8_t*)chunkUnC+(((((y%Zarr.get_chunks(1))*Zarr.get_chunks(0))+((z%Zarr.get_chunks(2))*Zarr.get_chunks(0)*Zarr.get_chunks(1)))+(endCoords[0]%Zarr.get_chunks(0)))*bytes),(uint8_t*)cRegion+(((((y%Zarr.get_chunks(1))*Zarr.get_chunks(0))+((z%Zarr.get_chunks(2))*Zarr.get_chunks(0)*Zarr.get_chunks(1)))+(endCoords[0]%Zarr.get_chunks(0)))*bytes),(Zarr.get_chunks(0)-(endCoords[0]%Zarr.get_chunks(0)))*bytes); + } + else{ + memset((uint8_t*)chunkUnC+((((y%Zarr.get_chunks(1))*Zarr.get_chunks(0))+((z%Zarr.get_chunks(2))*Zarr.get_chunks(0)*Zarr.get_chunks(1)))*bytes),0,(startCoords[0]%Zarr.get_chunks(0))*bytes); + memcpy((uint8_t*)chunkUnC+(((startCoords[0]%Zarr.get_chunks(0))+((y%Zarr.get_chunks(1))*Zarr.get_chunks(0))+((z%Zarr.get_chunks(2))*Zarr.get_chunks(0)*Zarr.get_chunks(1)))*bytes),(uint8_t*)zarrArr+((((cAV[0]*Zarr.get_chunks(0))-startCoords[0]+(startCoords[0]%Zarr.get_chunks(0)))+((y-startCoords[1])*writeShape[0])+((z-startCoords[2])*writeShape[0]*writeShape[1]))*bytes),((endCoords[0]%Zarr.get_chunks(0))-(startCoords[0]%Zarr.get_chunks(0)))*bytes); + memset((uint8_t*)chunkUnC+((((y%Zarr.get_chunks(1))*Zarr.get_chunks(0))+((z%Zarr.get_chunks(2))*Zarr.get_chunks(0)*Zarr.get_chunks(1)))+(endCoords[0]%Zarr.get_chunks(0))*bytes),0,(Zarr.get_chunks(0)-(endCoords[0]%Zarr.get_chunks(0)))*bytes); + } + } + else if((cAV[0]+1)*Zarr.get_chunks(0)>endCoords[0]){ + if(crop){ + memcpy((uint8_t*)chunkUnC+((((y%Zarr.get_chunks(1))*Zarr.get_chunks(0))+((z%Zarr.get_chunks(2))*Zarr.get_chunks(0)*Zarr.get_chunks(1)))*bytes),(uint8_t*)zarrArr+((((cAV[0]*Zarr.get_chunks(0))-startCoords[0])+((y-startCoords[1])*writeShape[0])+((z-startCoords[2])*writeShape[0]*writeShape[1]))*bytes),(endCoords[0]-(cAV[0]*Zarr.get_chunks(0)))*bytes); + + if((cAV[0]+1)*Zarr.get_chunks(0) > Zarr.get_shape(0)){ + memcpy((uint8_t*)chunkUnC+((((endCoords[0]-(cAV[0]*Zarr.get_chunks(0))))+((y%Zarr.get_chunks(1))*Zarr.get_chunks(0))+((z%Zarr.get_chunks(2))*Zarr.get_chunks(0)*Zarr.get_chunks(1)))*bytes),(uint8_t*)cRegion+((((endCoords[0]-(cAV[0]*Zarr.get_chunks(0))))+((y%Zarr.get_chunks(1))*Zarr.get_chunks(0))+((z%Zarr.get_chunks(2))*Zarr.get_chunks(0)*Zarr.get_chunks(1)))*bytes),(Zarr.get_shape(0)-endCoords[0])*bytes); + uint64_t xRest = ((cAV[0]+1)*Zarr.get_chunks(0))-Zarr.get_shape(0); + memset((uint8_t*)chunkUnC+(((Zarr.get_shape(0)-(cAV[0]*Zarr.get_chunks(0)))+((y%Zarr.get_chunks(1))*Zarr.get_chunks(0))+((z%Zarr.get_chunks(2))*Zarr.get_chunks(0)*Zarr.get_chunks(1)))*bytes),0,(xRest)*bytes); + } + else{ + memcpy((uint8_t*)chunkUnC+((((endCoords[0]-(cAV[0]*Zarr.get_chunks(0))))+((y%Zarr.get_chunks(1))*Zarr.get_chunks(0))+((z%Zarr.get_chunks(2))*Zarr.get_chunks(0)*Zarr.get_chunks(1)))*bytes),(uint8_t*)cRegion+((((endCoords[0]-(cAV[0]*Zarr.get_chunks(0))))+((y%Zarr.get_chunks(1))*Zarr.get_chunks(0))+((z%Zarr.get_chunks(2))*Zarr.get_chunks(0)*Zarr.get_chunks(1)))*bytes),(((cAV[0]+1)*Zarr.get_chunks(0))-endCoords[0])*bytes); + } + } + else{ + memcpy((uint8_t*)chunkUnC+((((y%Zarr.get_chunks(1))*Zarr.get_chunks(0))+((z%Zarr.get_chunks(2))*Zarr.get_chunks(0)*Zarr.get_chunks(1)))*bytes),(uint8_t*)zarrArr+((((cAV[0]*Zarr.get_chunks(0))-startCoords[0])+((y-startCoords[1])*writeShape[0])+((z-startCoords[2])*writeShape[0]*writeShape[1]))*bytes),(endCoords[0]%Zarr.get_chunks(0))*bytes); + memset((uint8_t*)chunkUnC+(((((y%Zarr.get_chunks(1))*Zarr.get_chunks(0))+((z%Zarr.get_chunks(2))*Zarr.get_chunks(0)*Zarr.get_chunks(1)))+(endCoords[0]%Zarr.get_chunks(0)))*bytes),0,(Zarr.get_chunks(0)-(endCoords[0]%Zarr.get_chunks(0)))*bytes); + } + } + else if((cAV[0]*Zarr.get_chunks(0)) < startCoords[0] && ((cAV[0]+1)*Zarr.get_chunks(0)) > startCoords[0]){ + if(crop){ + memcpy((uint8_t*)chunkUnC+((((y%Zarr.get_chunks(1))*Zarr.get_chunks(0))+((z%Zarr.get_chunks(2))*Zarr.get_chunks(0)*Zarr.get_chunks(1)))*bytes),(uint8_t*)cRegion+((((y%Zarr.get_chunks(1))*Zarr.get_chunks(0))+((z%Zarr.get_chunks(2))*Zarr.get_chunks(0)*Zarr.get_chunks(1)))*bytes),(startCoords[0]%Zarr.get_chunks(0))*bytes); + memcpy((uint8_t*)chunkUnC+(((startCoords[0]%Zarr.get_chunks(0))+((y%Zarr.get_chunks(1))*Zarr.get_chunks(0))+((z%Zarr.get_chunks(2))*Zarr.get_chunks(0)*Zarr.get_chunks(1)))*bytes),(uint8_t*)zarrArr+((((cAV[0]*Zarr.get_chunks(0))-startCoords[0]+(startCoords[0]%Zarr.get_chunks(0)))+((y-startCoords[1])*writeShape[0])+((z-startCoords[2])*writeShape[0]*writeShape[1]))*bytes),(Zarr.get_chunks(0)-(startCoords[0]%Zarr.get_chunks(0)))*bytes); + } + else{ + memset((uint8_t*)chunkUnC+((((y%Zarr.get_chunks(1))*Zarr.get_chunks(0))+((z%Zarr.get_chunks(2))*Zarr.get_chunks(0)*Zarr.get_chunks(1)))*bytes),0,(startCoords[0]%Zarr.get_chunks(0))*bytes); + memcpy((uint8_t*)chunkUnC+(((startCoords[0]%Zarr.get_chunks(0))+((y%Zarr.get_chunks(1))*Zarr.get_chunks(0))+((z%Zarr.get_chunks(2))*Zarr.get_chunks(0)*Zarr.get_chunks(1)))*bytes),(uint8_t*)zarrArr+((((cAV[0]*Zarr.get_chunks(0))-startCoords[0]+(startCoords[0]%Zarr.get_chunks(0)))+((y-startCoords[1])*writeShape[0])+((z-startCoords[2])*writeShape[0]*writeShape[1]))*bytes),(Zarr.get_chunks(0)-(startCoords[0]%Zarr.get_chunks(0)))*bytes); + } + } + } + else{ + memcpy((uint8_t*)chunkUnC+((((y%Zarr.get_chunks(1))*Zarr.get_chunks(0))+((z%Zarr.get_chunks(2))*Zarr.get_chunks(0)*Zarr.get_chunks(1)))*bytes),(uint8_t*)zarrArr+((((cAV[0]*Zarr.get_chunks(0))-startCoords[0])+((y-startCoords[1])*writeShape[0])+((z-startCoords[2])*writeShape[0]*writeShape[1]))*bytes),Zarr.get_chunks(0)*bytes); + } + } + } + } + else if (Zarr.get_order() == "C"){ + for(int64_t x = cAV[0]*Zarr.get_chunks(2); x < (cAV[0]+1)*Zarr.get_chunks(0); x++){ + for(int64_t y = cAV[1]*Zarr.get_chunks(1); y < (cAV[1]+1)*Zarr.get_chunks(1); y++){ + for(int64_t z = cAV[2]*Zarr.get_chunks(2); z < (cAV[2]+1)*Zarr.get_chunks(2); z++){ + switch(bytes){ + case 1: + if(x>=endCoords[0] || x= endCoords[1] || y=endCoords[2] || z=endCoords[0] || x= endCoords[1] || y=endCoords[2] || z=endCoords[0] || x= endCoords[1] || y=endCoords[2] || z=endCoords[0] || x= endCoords[1] || y=endCoords[2] || z(chunkC),csize); + file.close(); + rename(fileName.c_str(),fileNameFinal.c_str()); + } + else{ + std::ofstream file(fileName, std::ios::binary | std::ios::trunc); + if(!file.is_open()){ + #pragma omp critical + { + err = 1; + errString = "Check permissions or filepath. Cannot write to path: "+ + Zarr.get_fileName()+"\n"; + } + break; + } + file.write(reinterpret_cast(chunkC),csize); + file.close(); + } + free(cRegion); + } + free(chunkUnC); + free(chunkC); + + } + + if(err) mexErrMsgIdAndTxt("zarr:threadError",errString.c_str()); + + /* After using it, destroy the Blosc environment */ + //blosc_destroy(); +} + +// TODO: FIX MEMORY LEAKS +void mexFunction(int nlhs, mxArray *plhs[], + int nrhs, const mxArray *prhs[]) +{ + std::vector startCoords = {0,0,0}; + std::vector endCoords = {0,0,0}; + bool crop = false; + + // Dims are 1 by default + uint64_t iDims[3] = {1,1,1}; + + if(nrhs < 3) mexErrMsgIdAndTxt("zarr:inputError","This functions requires at least 3 arguments"); + else if(nrhs == 4 || nrhs == 5 || nrhs == 6){ + if(mxGetN(prhs[3]) == 6){ + crop = true; + startCoords[0] = (uint64_t)*(mxGetPr(prhs[3]))-1; + startCoords[1] = (uint64_t)*((mxGetPr(prhs[3])+1))-1; + startCoords[2] = (uint64_t)*((mxGetPr(prhs[3])+2))-1; + endCoords[0] = (uint64_t)*((mxGetPr(prhs[3])+3)); + endCoords[1] = (uint64_t)*((mxGetPr(prhs[3])+4)); + endCoords[2] = (uint64_t)*((mxGetPr(prhs[3])+5)); + + + uint64_t* iDimsT = (uint64_t*)mxGetDimensions(prhs[1]); + uint64_t niDims = (uint64_t) mxGetNumberOfDimensions(prhs[1]); + for(uint64_t i = 0; i < niDims; i++) iDims[i] = iDimsT[i]; + + if(startCoords[0]+1 < 1 || + startCoords[1]+1 < 1 || + startCoords[2]+1 < 1) mexErrMsgIdAndTxt("zarr:inputError","Lower bounds must be at least 1"); + + if(endCoords[0]-startCoords[0] > iDims[0] || + endCoords[1]-startCoords[1] > iDims[1] || + endCoords[2]-startCoords[2] > iDims[2]) mexErrMsgIdAndTxt("zarr:inputError","Bounds are invalid for the input data size"); + } + else if(mxGetN(prhs[3]) != 3) mexErrMsgIdAndTxt("zarr:inputError","Input range is not 6 or 3"); + } + else if (nrhs > 6) mexErrMsgIdAndTxt("zarr:inputError","Number of input arguments must be 4 or less"); + if(!mxIsChar(prhs[0])) mexErrMsgIdAndTxt("zarr:inputError","The first argument must be a string"); + std::string folderName(mxArrayToString(prhs[0])); + // Handle the tilde character in filenames on Linux/Mac + #ifndef _WIN32 + folderName = expandTilde(folderName.c_str()); + #endif + + // Check if metadata exists that we can use or if we have to create new metadata + zarr Zarr; + if(!fileExists(folderName+"/.zarray")){ + Zarr = zarr(); + Zarr.set_fileName(folderName); + } + else Zarr = zarr(folderName); + + + if(nrhs >= 5){ + Zarr.set_cname(mxArrayToString(prhs[4])); + } + if(nrhs == 6){ + if(mxGetN(prhs[5]) != 3) mexErrMsgIdAndTxt("zarr:inputError","subfolders must be an array of 3 numbers\n"); + Zarr.set_subfolders({(uint64_t)*(mxGetPr(prhs[5])), + (uint64_t)*((mxGetPr(prhs[5])+1)), + (uint64_t)*((mxGetPr(prhs[5])+2))}); + } + bool useUuid = (bool)*(mxGetPr(prhs[2])); + + void* zarrC = NULL; + + + mxClassID mDType = mxGetClassID(prhs[1]); + if(mDType == mxUINT8_CLASS){ + Zarr.set_dtype(" 3) mexErrMsgIdAndTxt("zarr:inputError","Input data must be 2D or 3D"); + + uint64_t* dims = (uint64_t*)mxGetDimensions(prhs[1]); + if(nDims == 3) Zarr.set_shape({dims[0],dims[1],dims[2]}); + else Zarr.set_shape({dims[0],dims[1],1}); + if(nrhs > 3){ + Zarr.set_chunks({(uint64_t)*(mxGetPr(prhs[3])), + (uint64_t)*((mxGetPr(prhs[3])+1)), + (uint64_t)*((mxGetPr(prhs[3])+2))}); + } + Zarr.write_zarray(); + } + else{ + Zarr.set_shape({endCoords[0],endCoords[1],endCoords[2]}); + + if(fileExists(folderName+"/.zarray")){ + if(endCoords[0]-startCoords[0] != iDims[0] || + endCoords[1]-startCoords[1] != iDims[1] || + endCoords[2]-startCoords[2] != iDims[2]) mexErrMsgIdAndTxt("zarr:inputError","Bounding box size does not match the size of the input data"); + } + else { + Zarr.write_zarray(); + } + + const std::string dtypeT(Zarr.get_dtype()); + Zarr = zarr(folderName); + if(dtypeT != Zarr.get_dtype()){ + uint64_t size = (endCoords[0]-startCoords[0])* + (endCoords[1]-startCoords[1])* + (endCoords[2]-startCoords[2]); + + uint64_t bitsT = 0; + if(dtypeT == " Zarr.get_shape(0) || + endCoords[1] > Zarr.get_shape(1) || + endCoords[2] > Zarr.get_shape(2)) mexErrMsgIdAndTxt("zarr:inputError","Upper bound is invalid"); + if(!crop){ + endCoords = {Zarr.get_shape(0),Zarr.get_shape(1),Zarr.get_shape(2)}; + startCoords = {0,0,0}; + } + const std::vector writeShape({endCoords[0]-startCoords[0], + endCoords[1]-startCoords[1], + endCoords[2]-startCoords[2]}); + //printf("%s startCoords[0]yz: %d %d %d endCoords[0]yz: %d %d %d chunkxyz: %d %d %d writeShape[0]yz: %d %d %d\n",Zarr.get_fileName().c_str(),startCoords[0],startCoords[1],startCoords[2],endCoords[0],endCoords[1],endCoords[2],Zarr.get_chunks(0),Zarr.get_chunks(1),Zarr.get_chunks(2),writeShape[0],writeShape[1],writeShape[2]); + + Zarr.set_chunkInfo(startCoords, endCoords); + if(Zarr.get_dtype() == " +#include +#include +#include +#ifdef _WIN32 +#include +#include +#else +#include +#endif +#include +#include +#include +#include "mex.h" +#include "zarr.h" +#include "helperfunctions.h" + +// Create a blank zarr object with default values +zarr::zarr() : +fileName(""), chunks({256,256,256}), blocksize(0), +clevel(5), cname("lz4"), id("blosc"), shuffle(1), dtype(">(); + try{ + // Try blosc compression types + cname = zarray.at("compressor").at("cname"); + clevel = zarray.at("compressor").at("clevel"); + blocksize = zarray.at("compressor").at("blocksize"); + id = zarray.at("compressor").at("id"); + shuffle = zarray.at("compressor").at("shuffle"); + } + catch(...){ + // Try gzip + clevel = zarray.at("compressor").at("level"); + cname = zarray.at("compressor").at("id"); + blocksize = 0; + id = ""; + shuffle = 0; + } + dtype = zarray.at("dtype"); + //fill_value = ""; + //filters = ""; + order = zarray.at("order"); + shape = zarray.at("shape").get>(); + zarr_format = zarray.at("zarr_format"); + } + catch(...){ + mexErrMsgIdAndTxt("zarr:zarrayError","Metadata is incomplete. Check the .zarray file"); + } + try{ + subfolders = zarray.at("subfolders").get>(); + } + catch(...){ + subfolders = {0,0,0}; + } +} + +// Create a new zarr file with .zarray metadata file (no actual data) +zarr::zarr(const std::string &fileName, const std::vector &chunks, + uint64_t blocksize, uint64_t clevel, const std::string &cname, + const std::string &id, uint64_t shuffle, const std::string &dtype, + const std::string &fill_value, const std::vector &filters, + const std::string &order, const std::vector &shape, + uint64_t zarr_format, const std::vector subfolders) : +fileName(fileName), chunks(chunks), blocksize(blocksize), +clevel(clevel), cname(cname), id(id), shuffle(shuffle), dtype(dtype), +fill_value(fill_value), filters(filters), order(order), shape(shape), +zarr_format(zarr_format), subfolders(subfolders) +{ + // Handle the tilde character in filenames on Linux/Mac + #ifndef _WIN32 + this->fileName = expandTilde(this->fileName.c_str()); + #endif + set_jsonValues(); +} + +zarr::~zarr(){ + +} + +// Write the current Metadata to the .zarray and create subfolders if needed +void zarr::write_zarray(){ + createSubfolders(); + set_jsonValues(); + write_jsonValues(); +} + +const std::string &zarr::get_fileName() const{ + return fileName; +} + +void zarr::set_fileName(const std::string &fileName){ + // Handle the tilde character in filenames on Linux/Mac + this->fileName = fileName; + #ifndef _WIN32 + this->fileName = expandTilde(this->fileName.c_str()); + #endif +} + +const uint64_t &zarr::get_chunks(const uint64_t &index) const{ + return chunks[index]; +} + +void zarr::set_chunks(const std::vector &chunks){ + this->chunks = chunks; +} + +const uint64_t &zarr::get_clevel() const{ + return clevel; +} + +void zarr::set_clevel(const uint64_t &clevel){ + this->clevel = clevel; +} + +const std::string &zarr::get_cname() const{ + return cname; +} + +void zarr::set_cname(const std::string &cname){ + this->cname = cname; +} + +const std::string &zarr::get_dtype() const{ + return dtype; +} + +void zarr::set_dtype(const std::string &dtype){ + this->dtype = dtype; +} + +const std::string &zarr::get_order() const{ + return order; +} + +void zarr::set_order(const std::string &order){ + this->order = order; +} + +const uint64_t &zarr::get_shape(const uint64_t &index) const{ + return shape[index]; +} + +void zarr::set_shape(const std::vector &shape){ + this->shape = shape; +} + +// Set the values of the JSON file to the current member values +void zarr::set_jsonValues(){ + zarray.clear(); + zarray["chunks"] = chunks; + + if(cname == "lz4" || cname == "blosclz" || cname == "lz4hc" || cname == "zlib" || cname == "zstd"){ + zarray["compressor"]["blocksize"] = blocksize; + zarray["compressor"]["clevel"] = clevel; + zarray["compressor"]["cname"] = cname; + zarray["compressor"]["id"] = id; + zarray["compressor"]["shuffle"] = shuffle; + } + else if(cname == "gzip"){ + zarray["compressor"]["id"] = id; + zarray["compressor"]["level"] = clevel; + } + else mexErrMsgIdAndTxt("zarr:zarrayError","Compressor: \"%s\" is not currently supported\n",cname.c_str()); + + // fill_value just 0 and filters null for now + zarray["dtype"] = dtype; + zarray["fill_value"] = 0; + zarray["filters"] = nullptr; + zarray["order"] = order; + + // zarr_format just 2 for now + zarray["shape"] = shape; + zarray["zarr_format"] = 2; + + zarray["subfolders"] = subfolders; +} + +// Write the current JSON to disk +void zarr::write_jsonValues(){ + // If the .zarray file does not exist then build the zarr fileName path recursively + const std::string fileNameFinal(fileName+"/.zarray"); + + //if(!std::filesystem::exists(fileNameFinal)){ + if(!fileExists(fileNameFinal)){ + mkdirRecursive(fileName.c_str()); + chmod(fileName.c_str(), 0775); + } + + const std::string uuid = generateUUID(); + const std::string fnFull(fileName+"/.zarray"+uuid); + + std::ofstream o(fnFull); + if(!o.good()) mexErrMsgIdAndTxt("zarr:zarrayError","Cannot open %s\n",fnFull.c_str()); + o << std::setw(4) << zarray << std::endl; + o.close(); + + rename(fnFull.c_str(),fileNameFinal.c_str()); +} + +const std::string zarr::get_subfoldersString(const std::vector &cAV) const{ + if(subfolders[0] == 0 && subfolders[1] == 0 && subfolders[2] == 0) return ""; + + std::vector currVals = {0,0,0}; + if(subfolders[0] > 0) currVals[0] = cAV[0]/subfolders[0]; + if(subfolders[1] > 0) currVals[1] = cAV[1]/subfolders[1]; + if(subfolders[2] > 0) currVals[2] = cAV[2]/subfolders[2]; + + return std::string(std::to_string(currVals[0])+"_"+ + std::to_string(currVals[1])+"_"+ + std::to_string(currVals[2])); +} + +void zarr::set_subfolders(const std::vector &subfolders){ + this->subfolders = subfolders; + zarray["subfolders"] = subfolders; +} + +// Fast ceiling for the subfolder function +uint64_t zarr::fastCeilDiv(uint64_t num, uint64_t denom){ + return 1 + ((num - 1) / denom); +} + +// Create subfolder "chunks" +void zarr::createSubfolders(){ + // If all elements are zero then we don't make subfolders + if(std::all_of(subfolders.begin(), + subfolders.end(), + [](int i){return !i;})) + { + return; + } + + std::vector nChunks = {fastCeilDiv(shape[0],chunks[0]), + fastCeilDiv(shape[1],chunks[1]), + fastCeilDiv(shape[2],chunks[2])}; + std::vector nSubfolders = {1,1,1}; + for(uint64_t i = 0; i < nSubfolders.size(); i++){ + if(subfolders[i] > 0){ + nSubfolders[i] = fastCeilDiv(nChunks[i],subfolders[i]); + } + } + + // Create subfolders + #pragma omp parallel for collapse(3) + for(uint64_t x = 0; x < nSubfolders[0]; x++){ + for(uint64_t y = 0; y < nSubfolders[1]; y++){ + for(uint64_t z = 0; z < nSubfolders[2]; z++){ + std::string currName(fileName+"/"+std::to_string(x)+"_"+ + std::to_string(y)+"_"+std::to_string(z)); + mkdirRecursive(currName.c_str()); + } + } + } +} + +const std::vector zarr::get_chunkAxisVals(const std::string &fileName) const{ + std::vector cAV(3); + char* ptr; + cAV[0] = strtol(fileName.c_str(), &ptr, 10); + ptr++; + cAV[1] = strtol(ptr, &ptr, 10); + ptr++; + cAV[2] = strtol(ptr, &ptr, 10); + return cAV; +} + +void zarr::set_chunkInfo(const std::vector &startCoords, + const std::vector &endCoords) +{ + + uint64_t xStartAligned = startCoords[0]-(startCoords[0]%chunks[0]); + uint64_t yStartAligned = startCoords[1]-(startCoords[1]%chunks[1]); + uint64_t zStartAligned = startCoords[2]-(startCoords[2]%chunks[2]); + uint64_t xStartChunk = (xStartAligned/chunks[0]); + uint64_t yStartChunk = (yStartAligned/chunks[1]); + uint64_t zStartChunk = (zStartAligned/chunks[2]); + + uint64_t xEndAligned = endCoords[0]; + uint64_t yEndAligned = endCoords[1]; + uint64_t zEndAligned = endCoords[2]; + + if(xEndAligned%chunks[0]) xEndAligned = endCoords[0]-(endCoords[0]%chunks[0])+chunks[0]; + if(yEndAligned%chunks[1]) yEndAligned = endCoords[1]-(endCoords[1]%chunks[1])+chunks[1]; + if(zEndAligned%chunks[2]) zEndAligned = endCoords[2]-(endCoords[2]%chunks[2])+chunks[2]; + uint64_t xEndChunk = (xEndAligned/chunks[0]); + uint64_t yEndChunk = (yEndAligned/chunks[1]); + uint64_t zEndChunk = (zEndAligned/chunks[2]); + + uint64_t xChunks = (xEndChunk-xStartChunk); + uint64_t yChunks = (yEndChunk-yStartChunk); + uint64_t zChunks = (zEndChunk-zStartChunk); + numChunks = xChunks*yChunks*zChunks; + chunkNames = std::vector(numChunks); + #pragma omp parallel for collapse(3) + for(uint64_t x = xStartChunk; x < xEndChunk; x++){ + for(uint64_t y = yStartChunk; y < yEndChunk; y++){ + for(uint64_t z = zStartChunk; z < zEndChunk; z++){ + uint64_t currFile = (z-zStartChunk)+((y-yStartChunk)*zChunks)+((x-xStartChunk)*yChunks*zChunks); + chunkNames[currFile] = std::to_string(x)+"."+std::to_string(y)+"."+std::to_string(z); + } + } + } +} + +const std::string &zarr::get_chunkNames(const uint64_t &index) const{ + return chunkNames[index]; +} + +const uint64_t &zarr::get_numChunks() const{ + return numChunks; +} \ No newline at end of file diff --git a/mexSrc/zarr.h b/mexSrc/zarr.h new file mode 100644 index 0000000..10448cd --- /dev/null +++ b/mexSrc/zarr.h @@ -0,0 +1,68 @@ +#ifndef ZARR_H +#define ZARR_H +#include +#include +#include +#include +using json = nlohmann::json; + +class zarr +{ +public: + zarr(); + zarr(const std::string &fileName); + zarr(const std::string &fileName, const std::vector &chunks, + uint64_t blocksize, uint64_t clevel, const std::string &cname, + const std::string &id, uint64_t shuffle, const std::string &dtype, + const std::string &fill_value, const std::vector &filters, + const std::string &order, const std::vector &shape, + uint64_t zarr_format, const std::vector subfolders); + ~zarr(); + void write_zarray(); + const std::string &get_fileName() const; + void set_fileName(const std::string &fileName); + const uint64_t &get_chunks(const uint64_t &index) const; + void set_chunks(const std::vector &chunks); + const uint64_t &get_clevel() const; + void set_clevel(const uint64_t &clevel); + const std::string &get_cname() const; + void set_cname(const std::string &cname); + const std::string &get_dtype() const; + void set_dtype(const std::string &dtype); + const std::string &get_order() const; + void set_order(const std::string &order); + const uint64_t &get_shape(const uint64_t &index) const; + void set_shape(const std::vector &shape); + const std::string get_subfoldersString(const std::vector &cAV) const; + void set_subfolders(const std::vector &subfolders); + const std::vector get_chunkAxisVals(const std::string &fileName) const; + void set_chunkInfo(const std::vector &startCoords, + const std::vector &endCoords); + const std::string &get_chunkNames(const uint64_t &index) const; + const uint64_t &get_numChunks() const; + //static +private: + void set_jsonValues(); + void write_jsonValues(); + uint64_t fastCeilDiv(uint64_t num, uint64_t denom); + void createSubfolders(); + std::string fileName; + json zarray; + std::vector chunks; + uint64_t blocksize; + uint64_t clevel; + std::string cname; + std::string id; + uint64_t shuffle; + std::string dtype; + std::string fill_value; + std::vector filters; + std::string order; + std::vector shape; + uint64_t zarr_format; + std::vector subfolders; + + std::vector chunkNames; + uint64_t numChunks; +}; +#endif \ No newline at end of file diff --git a/parallelReadZarr/linux/libblosc2.so b/parallelReadZarr/linux/libblosc2.so deleted file mode 120000 index a03f65e..0000000 --- a/parallelReadZarr/linux/libblosc2.so +++ /dev/null @@ -1 +0,0 @@ -libblosc2.so.2 \ No newline at end of file diff --git a/parallelReadZarr/linux/libblosc2.so.2 b/parallelReadZarr/linux/libblosc2.so.2 deleted file mode 120000 index ff853f6..0000000 --- a/parallelReadZarr/linux/libblosc2.so.2 +++ /dev/null @@ -1 +0,0 @@ -libblosc2.so.2.0.5 \ No newline at end of file diff --git a/parallelReadZarr/linux/libblosc2.so.2.0.5 b/parallelReadZarr/linux/libblosc2.so.2.0.5 deleted file mode 100755 index 308d3a7..0000000 Binary files a/parallelReadZarr/linux/libblosc2.so.2.0.5 and /dev/null differ diff --git a/parallelReadZarr/linux/libcjson.so b/parallelReadZarr/linux/libcjson.so deleted file mode 120000 index 0026d72..0000000 --- a/parallelReadZarr/linux/libcjson.so +++ /dev/null @@ -1 +0,0 @@ -libcjson.so.1 \ No newline at end of file diff --git a/parallelReadZarr/linux/libcjson.so.1 b/parallelReadZarr/linux/libcjson.so.1 deleted file mode 120000 index ee90626..0000000 --- a/parallelReadZarr/linux/libcjson.so.1 +++ /dev/null @@ -1 +0,0 @@ -libcjson.so.1.7.15 \ No newline at end of file diff --git a/parallelReadZarr/linux/libcjson.so.1.7.15 b/parallelReadZarr/linux/libcjson.so.1.7.15 deleted file mode 100755 index eb168c2..0000000 Binary files a/parallelReadZarr/linux/libcjson.so.1.7.15 and /dev/null differ diff --git a/parallelReadZarr/linux/libgomp.so b/parallelReadZarr/linux/libgomp.so deleted file mode 100755 index eccfabc..0000000 Binary files a/parallelReadZarr/linux/libgomp.so and /dev/null differ diff --git a/parallelReadZarr/linux/libgomp.so.1 b/parallelReadZarr/linux/libgomp.so.1 deleted file mode 100755 index eccfabc..0000000 Binary files a/parallelReadZarr/linux/libgomp.so.1 and /dev/null differ diff --git a/parallelReadZarr/linux/libgomp.so.1.0.0 b/parallelReadZarr/linux/libgomp.so.1.0.0 deleted file mode 100755 index eccfabc..0000000 Binary files a/parallelReadZarr/linux/libgomp.so.1.0.0 and /dev/null differ diff --git a/parallelReadZarr/linux/libz.so b/parallelReadZarr/linux/libz.so deleted file mode 120000 index 95e0ebd..0000000 --- a/parallelReadZarr/linux/libz.so +++ /dev/null @@ -1 +0,0 @@ -libz.so.1.2.11 \ No newline at end of file diff --git a/parallelReadZarr/linux/libz.so.1 b/parallelReadZarr/linux/libz.so.1 deleted file mode 120000 index 95e0ebd..0000000 --- a/parallelReadZarr/linux/libz.so.1 +++ /dev/null @@ -1 +0,0 @@ -libz.so.1.2.11 \ No newline at end of file diff --git a/parallelReadZarr/linux/libz.so.1.2.11 b/parallelReadZarr/linux/libz.so.1.2.11 deleted file mode 100755 index b377c07..0000000 Binary files a/parallelReadZarr/linux/libz.so.1.2.11 and /dev/null differ diff --git a/parallelReadZarr/linux/parallelReadZarr.mexa64 b/parallelReadZarr/linux/parallelReadZarr.mexa64 deleted file mode 100755 index 50e01bc..0000000 Binary files a/parallelReadZarr/linux/parallelReadZarr.mexa64 and /dev/null differ diff --git a/parallelReadZarr/parallelReadZarr.c b/parallelReadZarr/parallelReadZarr.c deleted file mode 100755 index 46946cf..0000000 --- a/parallelReadZarr/parallelReadZarr.c +++ /dev/null @@ -1,580 +0,0 @@ -#include -#include -#include -#include "blosc2.h" -#include "cjson/cJSON.h" -#include -#include -#include "mex.h" - -#include "zlib.h" -// mex -v COPTIMFLAGS="-O3 -DNDEBUG" LDOPTIMFLAGS="-Wl',-rpath='''$ORIGIN'''' -O3 -DNDEBUG" CFLAGS='$CFLAGS -O3 -fopenmp' LDFLAGS='$LDFLAGS -O3 -fopenmp' '-I/global/home/groups/software/sl-7.x86_64/modules/cBlosc/2.0.4/include/' '-I/global/home/groups/software/sl-7.x86_64/modules/cJSON/1.7.15/include/' '-L/global/home/groups/software/sl-7.x86_64/modules/cBlosc/2.0.4/lib64' -lblosc2 -L'/global/home/groups/software/sl-7.x86_64/modules/cJSON/1.7.15/lib64' -lcjson -lz parallelReadZarr.c - -// Handle the tilde character in filenames on Linux/Mac -#ifndef _WIN32 -#include -char* expandTilde(char* path) { - wordexp_t expPath; - wordexp(path, &expPath, 0); - return expPath.we_wordv[0]; -} -#endif - -const char fileSep = -#ifdef _WIN32 - '\\'; -#else - '/'; -#endif - -#ifdef _WIN32 -char* strndup (const char *s, size_t n) -{ - size_t len = strnlen (s, n); - char *new = (char *) malloc (len + 1); - if (new == NULL) - return NULL; - new[len] = '\0'; - return (char *) memcpy (new, s, len); -} - -int _vscprintf_so(const char * format, va_list pargs) { - int retval; - va_list argcopy; - va_copy(argcopy, pargs); - retval = vsnprintf(NULL, 0, format, argcopy); - va_end(argcopy); - return retval; -} - -int vasprintf(char **strp, const char *fmt, va_list ap) { - int len = _vscprintf_so(fmt, ap); - if (len == -1) return -1; - char *str = malloc((size_t) len + 1); - if (!str) return -1; - int r = vsnprintf(str, len + 1, fmt, ap); /* "secure" version of vsprintf */ - if (r == -1) return free(str), -1; - *strp = str; - return r; -} - -int asprintf(char *strp[], const char *fmt, ...) { - va_list ap; - va_start(ap, fmt); - int r = vasprintf(strp, fmt, ap); - va_end(ap); - return r; -} -#endif -/* -void decompErr(){ - #pragma omp critical - { - err = 1; - sprintf(errString,"Decompression error. Error code: %d ChunkName: %s/%s\n",uncErr,folderName,cI.chunkNames[f]); - } -}*/ - -void* mallocDynamic(uint64_t x, uint64_t bits){ - switch(bits){ - case 8: - return malloc(x*sizeof(uint8_t)); - case 16: - return malloc(x*sizeof(uint16_t)); - case 32: - return malloc(x*sizeof(float)); - case 64: - return malloc(x*sizeof(double)); - default: - printf("Image is not 8/16 bit, single, or double. Using single."); - return malloc(x*sizeof(float)); - } -} - -struct chunkInfo{ - char** chunkNames; - int64_t numChunks; -}; - -struct chunkAxisVals{ - uint64_t x; - uint64_t y; - uint64_t z; -}; - -struct chunkAxisVals getChunkAxisVals(char* fileName){ - struct chunkAxisVals cAV; - char* ptr; - cAV.x = strtol(fileName, &ptr, 10); - ptr++; - cAV.y = strtol(ptr, &ptr, 10); - ptr++; - cAV.z = strtol(ptr, &ptr, 10); - return cAV; -} - -struct chunkInfo getChunkInfo(char* folderName, uint64_t startX, uint64_t startY, uint64_t startZ, uint64_t endX, uint64_t endY,uint64_t endZ,uint64_t chunkXSize,uint64_t chunkYSize,uint64_t chunkZSize){ - struct chunkInfo cI; - cI.numChunks = 0; - cI.chunkNames = NULL; - - uint64_t xStartAligned = startX-(startX%chunkXSize); - uint64_t yStartAligned = startY-(startY%chunkYSize); - uint64_t zStartAligned = startZ-(startZ%chunkZSize); - uint64_t xStartChunk = (xStartAligned/chunkXSize); - uint64_t yStartChunk = (yStartAligned/chunkYSize); - uint64_t zStartChunk = (zStartAligned/chunkZSize); - - uint64_t xEndAligned = endX; - uint64_t yEndAligned = endY; - uint64_t zEndAligned = endZ; - - if(xEndAligned%chunkXSize) xEndAligned = endX-(endX%chunkXSize)+chunkXSize; - if(yEndAligned%chunkYSize) yEndAligned = endY-(endY%chunkYSize)+chunkYSize; - if(zEndAligned%chunkZSize) zEndAligned = endZ-(endZ%chunkZSize)+chunkZSize; - uint64_t xEndChunk = (xEndAligned/chunkXSize); - uint64_t yEndChunk = (yEndAligned/chunkYSize); - uint64_t zEndChunk = (zEndAligned/chunkZSize); - - uint64_t xChunks = (xEndChunk-xStartChunk); - uint64_t yChunks = (yEndChunk-yStartChunk); - uint64_t zChunks = (zEndChunk-zStartChunk); - - uint64_t file_count = xChunks*yChunks*zChunks; - - char** chunkNames = malloc(file_count*sizeof(char*)); - #pragma omp parallel for collapse(3) - for(uint64_t x = xStartChunk; x < xEndChunk; x++){ - for(uint64_t y = yStartChunk; y < yEndChunk; y++){ - for(uint64_t z = zStartChunk; z < zEndChunk; z++){ - uint64_t currFile = (z-zStartChunk)+((y-yStartChunk)*zChunks)+((x-xStartChunk)*yChunks*zChunks); - asprintf(&chunkNames[currFile],"%llu.%llu.%llu",x,y,z); - } - } - } - cI.chunkNames = chunkNames; - cI.numChunks = file_count; - return cI; -} - -char* getSubfolderString(struct chunkAxisVals *cAV, uint64_t subfolderSizeX, uint64_t subfolderSizeY, uint64_t subfolderSizeZ){ - if(subfolderSizeX == 0 && subfolderSizeY == 0 && subfolderSizeZ == 0) return NULL; - - uint64_t currX = 0; - uint64_t currY = 0; - uint64_t currZ = 0; - if(subfolderSizeX > 0) currX = cAV->x/subfolderSizeX; - if(subfolderSizeY > 0) currY = cAV->y/subfolderSizeY; - if(subfolderSizeZ > 0) currZ = cAV->z/subfolderSizeZ; - - char* currName = NULL; - asprintf(&currName,"%llu_%llu_%llu",currX,currY,currZ); - return currName; - -} - -void setChunkShapeFromJSON(cJSON *json, uint64_t *x, uint64_t *y, uint64_t *z){ - *x = json->child->valueint; - *y = json->child->next->valueint; - *z = json->child->next->next->valueint; -} - -void setDTypeFromJSON(cJSON *json, char* dtype){ - dtype[0] = json->valuestring[0]; - dtype[1] = json->valuestring[1]; - dtype[2] = json->valuestring[2]; - dtype[3] = json->valuestring[3]; -} - -void setOrderFromJSON(cJSON *json, char* order){ - *order = json->valuestring[0]; -} - -void setShapeFromJSON(cJSON *json, uint64_t *x, uint64_t *y, uint64_t *z){ - *x = json->child->valueint; - *y = json->child->next->valueint; - *z = json->child->next->next->valueint; -} -void setCnameFromJSON(cJSON *json, char** cname){ - cJSON *jsonItem = json->child; - - - while(jsonItem){ - if(!strcmp(jsonItem->string,"cname")){ - *cname = strdup(jsonItem->valuestring); - return; - } - // For gzip - if(!strcmp(jsonItem->string,"id") && !strcmp(jsonItem->valuestring,"gzip")){ - *cname = strdup(jsonItem->valuestring); - return; - } - jsonItem = jsonItem->next; - } - mexErrMsgIdAndTxt("zarr:zarrayError","Compressor: \"%s\" is not currently supported\n",*cname); - -} - -void setSubfoldersFromJSON(cJSON *json, uint64_t *subfolderSizeX, uint64_t *subfolderSizeY, uint64_t *subfolderSizeZ){ - *subfolderSizeX = json->child->valueint; - *subfolderSizeY = json->child->next->valueint; - *subfolderSizeZ = json->child->next->next->valueint; -} - -void setValuesFromJSON(char* fileName,uint64_t *chunkXSize,uint64_t *chunkYSize,uint64_t *chunkZSize,char* dtype,char* order,uint64_t *shapeX,uint64_t *shapeY,uint64_t *shapeZ,char** cname, uint64_t *subfolderSizeX, uint64_t *subfolderSizeY, uint64_t *subfolderSizeZ){ - - char* zArray = ".zarray"; - char* fnFull = (char*)malloc(strlen(fileName)+9); - fnFull[0] = '\0'; - char fileSepS[2]; - fileSepS[0] = fileSep; - fileSepS[1] = '\0'; - - strcat(fnFull,fileName); - strcat(fnFull,fileSepS); - strcat(fnFull,zArray); - - FILE *fileptr = fopen(fnFull, "rb"); - if(!fileptr) mexErrMsgIdAndTxt("zarr:inputError","Failed to open JSON File: %s\n",fnFull); - free(fnFull); - - fseek(fileptr, 0, SEEK_END); - long filelen = ftell(fileptr); - rewind(fileptr); - char* buffer = (char *)malloc((filelen)); - fread(buffer, filelen, 1, fileptr); - fclose(fileptr); - cJSON *json = cJSON_ParseWithLength(buffer,filelen); - - while(json){ - if(!json->string){ - json = json->child; - continue; - } - else if(!strcmp(json->string,"chunks")){ - setChunkShapeFromJSON(json, chunkXSize,chunkYSize,chunkZSize); - } - else if(!strcmp(json->string,"dtype")){ - setDTypeFromJSON(json, dtype); - } - else if(!strcmp(json->string,"order")){ - setOrderFromJSON(json, order); - } - else if(!strcmp(json->string,"shape")){ - setShapeFromJSON(json, shapeX,shapeY,shapeZ); - } - else if(!strcmp(json->string,"compressor")){ - setCnameFromJSON(json, cname); - } - else if(!strcmp(json->string,"subfolders")){ - setSubfoldersFromJSON(json, subfolderSizeX, subfolderSizeY, subfolderSizeZ); - } - json = json->next; - } - cJSON_Delete(json); -} - -void parallelReadZarrMex(void* zarr, char* folderName,uint64_t startX, uint64_t startY, uint64_t startZ, uint64_t endX, uint64_t endY,uint64_t endZ,uint64_t chunkXSize,uint64_t chunkYSize,uint64_t chunkZSize,uint64_t shapeX,uint64_t shapeY,uint64_t shapeZ, uint64_t bits, char order, char* cname, uint64_t subfolderSizeX, uint64_t subfolderSizeY, uint64_t subfolderSizeZ){ - uint64_t bytes = (bits/8); - char fileSepS[2]; - fileSepS[0] = fileSep; - fileSepS[1] = '\0'; - - /* Initialize the Blosc compressor */ - int32_t numWorkers = omp_get_max_threads(); - blosc_init(); - blosc_set_nthreads(numWorkers); - - struct chunkInfo cI = getChunkInfo(folderName, startX, startY, startZ, endX, endY, endZ,chunkXSize,chunkYSize,chunkZSize); - if(!cI.chunkNames) mexErrMsgIdAndTxt("zarr:inputError","File \"%s\" cannot be opened",folderName); - - int32_t batchSize = (cI.numChunks-1)/numWorkers+1; - uint64_t s = chunkXSize*chunkYSize*chunkZSize; - uint64_t sB = s*bytes; - int32_t w; - int err = 0; - char errString[10000]; - - - #pragma omp parallel for if(numWorkers<=cI.numChunks) - for(w = 0; w < numWorkers; w++){ - /* - if(strcmp(cname,"gzip")){ - bufferDest = mallocDynamic(s,bits); - } - else{ - bufferDest = calloc(sB,1); - } - */ - void* bufferDest = mallocDynamic(s,bits); - uint64_t lastFileLen = 0; - char *buffer = NULL; - for(int64_t f = w*batchSize; f < (w+1)*batchSize; f++){ - if(f>=cI.numChunks || err) break; - struct chunkAxisVals cAV = getChunkAxisVals(cI.chunkNames[f]); - char* subfolderName = getSubfolderString(&cAV,subfolderSizeX,subfolderSizeY,subfolderSizeZ); - if(!subfolderName){ - subfolderName = malloc(1); - subfolderName[0] = '\0'; - } - //malloc +2 for null term and filesep - char *fileName = malloc(strlen(folderName)+1+strlen(subfolderName)+1+strlen(cI.chunkNames[f])+1); - fileName[0] = '\0'; - strcat(fileName,folderName); - strcat(fileName,fileSepS); - strcat(fileName,subfolderName); - strcat(fileName,fileSepS); - strcat(fileName,cI.chunkNames[f]); - - FILE *fileptr = fopen(fileName, "rb"); - if(!fileptr){ - #pragma omp critical - { - memset(zarr,0,sB); - free(fileName); - //err = 1; - //sprintf(errString,"Could not open file: %s\n",fileName); - } - break; - } - free(fileName); - free(subfolderName); - - fseek(fileptr, 0, SEEK_END); - long filelen = ftell(fileptr); - rewind(fileptr); - if(lastFileLen < filelen){ - free(buffer); - buffer = (char*) malloc(filelen); - lastFileLen = filelen; - } - fread(buffer, filelen, 1, fileptr); - fclose(fileptr); - - - // Decompress - int64_t dsize = -1; - int uncErr = 0; - if(strcmp(cname,"gzip")){ - dsize = blosc2_decompress(buffer, filelen, bufferDest, sB); - } - else{ - dsize = sB; - z_stream stream; - stream.zalloc = Z_NULL; - stream.zfree = Z_NULL; - stream.opaque = Z_NULL; - stream.avail_in = (uInt)filelen; - stream.avail_out = (uInt)dsize; - while(stream.avail_in > 0){ - - dsize = sB; - - stream.next_in = (uint8_t*)buffer+(filelen-stream.avail_in); - stream.next_out = (uint8_t*)bufferDest+(sB-stream.avail_out); - - uncErr = inflateInit2(&stream, 32); - if(uncErr){ - #pragma omp critical - { - err = 1; - sprintf(errString,"Decompression error. Error code: %d ChunkName: %s/%s\n",uncErr,folderName,cI.chunkNames[f]); - } - break; - } - - uncErr = inflate(&stream, Z_NO_FLUSH); - - if(uncErr != Z_STREAM_END){ - #pragma omp critical - { - err = 1; - sprintf(errString,"Decompression error. Error code: %d ChunkName: %s/%s\n",uncErr,folderName,cI.chunkNames[f]); - } - break; - } - } - if(inflateEnd(&stream)){ - #pragma omp critical - { - err = 1; - sprintf(errString,"Decompression error. Error code: %d ChunkName: %s/%s\n",uncErr,folderName,cI.chunkNames[f]); - } - break; - } - } - - - if(dsize < 0){ - #pragma omp critical - { - err = 1; - sprintf(errString,"Decompression error. Error code: %d ChunkName: %s/%s\n",dsize,folderName,cI.chunkNames[f]); - } - break; - } - - //printf("ChunkName: %s\n",cI.chunkNames[f]); - //printf("w: %d b: %d\n",w,f); - if(order == 'F'){ - for(int64_t z = cAV.z*chunkZSize; z < (cAV.z+1)*chunkZSize; z++){ - if(z>=endZ) break; - else if(z=endY) break; - else if(y startX) || (cAV.x+1)*chunkXSize>endX){ - if(((cAV.x*chunkXSize) < startX && ((cAV.x+1)*chunkXSize) > startX) && (cAV.x+1)*chunkXSize>endX){ - memcpy((uint8_t*)zarr+((((cAV.x*chunkXSize)-startX+(startX%chunkXSize))+((y-startY)*shapeX)+((z-startZ)*shapeX*shapeY))*bytes),(uint8_t*)bufferDest+(((startX%chunkXSize)+((y%chunkYSize)*chunkXSize)+((z%chunkZSize)*chunkXSize*chunkYSize))*bytes),((endX%chunkXSize)-(startX%chunkXSize))*bytes); - } - else if((cAV.x+1)*chunkXSize>endX){ - memcpy((uint8_t*)zarr+((((cAV.x*chunkXSize)-startX)+((y-startY)*shapeX)+((z-startZ)*shapeX*shapeY))*bytes),(uint8_t*)bufferDest+((((y%chunkYSize)*chunkXSize)+((z%chunkZSize)*chunkXSize*chunkYSize))*bytes),(endX%chunkXSize)*bytes); - } - else if((cAV.x*chunkXSize) < startX && ((cAV.x+1)*chunkXSize) > startX){ - memcpy((uint8_t*)zarr+((((cAV.x*chunkXSize-startX+(startX%chunkXSize)))+((y-startY)*shapeX)+((z-startZ)*shapeX*shapeY))*bytes),(uint8_t*)bufferDest+(((startX%chunkXSize)+((y%chunkYSize)*chunkXSize)+((z%chunkZSize)*chunkXSize*chunkYSize))*bytes),(chunkXSize-(startX%chunkXSize))*bytes); - } - } - else{ - memcpy((uint8_t*)zarr+((((cAV.x*chunkXSize)-startX)+((y-startY)*shapeX)+((z-startZ)*shapeX*shapeY))*bytes),(uint8_t*)bufferDest+((((y%chunkYSize)*chunkXSize)+((z%chunkZSize)*chunkXSize*chunkYSize))*bytes),chunkXSize*bytes); - } - } - } - - } - else if (order == 'C'){ - for(int64_t x = cAV.x*chunkXSize; x < (cAV.x+1)*chunkXSize; x++){ - if(x>=endX) break; - else if(x=endY) break; - else if(y=endZ) break; - else if(z 2) mexErrMsgIdAndTxt("zarr:inputError","Number of input arguments must be 1 or 2"); - if(!mxIsChar(prhs[0])) mexErrMsgIdAndTxt("zarr:inputError","The first argument must be a string"); - char* folderName = mxArrayToString(prhs[0]); - - // Handle the tilde character in filenames on Linux/Mac - #ifndef _WIN32 - if(strchr(folderName,'~')) folderName = expandTilde(folderName); - #endif - - uint64_t shapeX = 0; - uint64_t shapeY = 0; - uint64_t shapeZ = 0; - uint64_t chunkXSize = 0; - uint64_t chunkYSize = 0; - uint64_t chunkZSize = 0; - char dtype[4]; - char order; - - setValuesFromJSON(folderName,&chunkXSize,&chunkYSize,&chunkZSize,dtype,&order,&shapeX,&shapeY,&shapeZ,&cname,&subfolderSizeX,&subfolderSizeY,&subfolderSizeZ); - if(endX > shapeX || endY > shapeY || endZ > shapeZ) mexErrMsgIdAndTxt("zarr:inputError","Upper bound is invalid"); - if(nrhs == 1){ - endX = shapeX; - endY = shapeY; - endZ = shapeZ; - } - uint64_t dim[3]; - shapeX = endX-startX; - shapeY = endY-startY; - shapeZ = endZ-startZ; - dim[0] = shapeX; - dim[1] = shapeY; - dim[2] = shapeZ; - - - - if(dtype[1] == 'u' && dtype[2] == '1'){ - uint64_t bits = 8; - plhs[0] = mxCreateNumericArray(3,dim,mxUINT8_CLASS, mxREAL); - uint8_t* zarr = (uint8_t*)mxGetPr(plhs[0]); - parallelReadZarrMex((void*)zarr,folderName,startX,startY,startZ,endX,endY,endZ,chunkXSize,chunkYSize,chunkZSize,shapeX,shapeY,shapeZ,bits,order,cname,subfolderSizeX,subfolderSizeY,subfolderSizeZ); - } - else if(dtype[1] == 'u' && dtype[2] == '2'){ - uint64_t bits = 16; - plhs[0] = mxCreateNumericArray(3,dim,mxUINT16_CLASS, mxREAL); - uint16_t* zarr = (uint16_t*)mxGetPr(plhs[0]); - parallelReadZarrMex((void*)zarr,folderName,startX,startY,startZ,endX,endY,endZ,chunkXSize,chunkYSize,chunkZSize,shapeX,shapeY,shapeZ,bits,order,cname,subfolderSizeX,subfolderSizeY,subfolderSizeZ); - } - else if(dtype[1] == 'f' && dtype[2] == '4'){ - uint64_t bits = 32; - plhs[0] = mxCreateNumericArray(3,dim,mxSINGLE_CLASS, mxREAL); - float* zarr = (float*)mxGetPr(plhs[0]); - parallelReadZarrMex((void*)zarr,folderName,startX,startY,startZ,endX,endY,endZ,chunkXSize,chunkYSize,chunkZSize,shapeX,shapeY,shapeZ,bits,order,cname,subfolderSizeX,subfolderSizeY,subfolderSizeZ); - } - else if(dtype[1] == 'f' && dtype[2] == '8'){ - uint64_t bits = 64; - plhs[0] = mxCreateNumericArray(3,dim,mxDOUBLE_CLASS, mxREAL); - double* zarr = (double*)mxGetPr(plhs[0]); - parallelReadZarrMex((void*)zarr,folderName,startX,startY,startZ,endX,endY,endZ,chunkXSize,chunkYSize,chunkZSize,shapeX,shapeY,shapeZ,bits,order,cname,subfolderSizeX,subfolderSizeY,subfolderSizeZ); - } - else{ - mexErrMsgIdAndTxt("tiff:dataTypeError","Data type not suppported"); - } - -} diff --git a/parallelReadZarr/windows/cjson.dll b/parallelReadZarr/windows/cjson.dll deleted file mode 100755 index 4e3efdc..0000000 Binary files a/parallelReadZarr/windows/cjson.dll and /dev/null differ diff --git a/parallelReadZarr/windows/libblosc2.dll b/parallelReadZarr/windows/libblosc2.dll deleted file mode 100755 index ad8e48b..0000000 Binary files a/parallelReadZarr/windows/libblosc2.dll and /dev/null differ diff --git a/parallelReadZarr/windows/parallelReadZarr.mexw64 b/parallelReadZarr/windows/parallelReadZarr.mexw64 deleted file mode 100755 index 6593fa1..0000000 Binary files a/parallelReadZarr/windows/parallelReadZarr.mexw64 and /dev/null differ diff --git a/parallelWriteZarr/helperFunctions.c b/parallelWriteZarr/helperFunctions.c deleted file mode 100755 index b0d0e3f..0000000 --- a/parallelWriteZarr/helperFunctions.c +++ /dev/null @@ -1,450 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include -#ifdef _WIN32 -#include -#include -#else -#include -#endif -#include -#include "mex.h" -#include "helperFunctions.h" - -#ifndef _WIN32 -#include -char* expandTilde(char* path) { - wordexp_t expPath; - wordexp(path, &expPath, 0); - return expPath.we_wordv[0]; -} -#endif - -const char fileSep = -#ifdef _WIN32 - '\\'; -#else - '/'; -#endif - -#ifdef _WIN32 -char* strndup (const char *s, size_t n) -{ - size_t len = strnlen (s, n); - char *new = (char *) malloc (len + 1); - if (new == NULL) - return NULL; - new[len] = '\0'; - return (char *) memcpy (new, s, len); -} - -int _vscprintf_so(const char * format, va_list pargs) { - int retval; - va_list argcopy; - va_copy(argcopy, pargs); - retval = vsnprintf(NULL, 0, format, argcopy); - va_end(argcopy); - return retval; -} - -int vasprintf(char **strp, const char *fmt, va_list ap) { - int len = _vscprintf_so(fmt, ap); - if (len == -1) return -1; - char *str = malloc((size_t) len + 1); - if (!str) return -1; - int r = vsnprintf(str, len + 1, fmt, ap); /* "secure" version of vsprintf */ - if (r == -1) return free(str), -1; - *strp = str; - return r; -} - -int asprintf(char *strp[], const char *fmt, ...) { - va_list ap; - va_start(ap, fmt); - int r = vasprintf(strp, fmt, ap); - va_end(ap); - return r; -} -#endif - -static void mkdirRecursive(const char *dir) { - char tmp[8192]; - char *p = NULL; - size_t len; - #ifdef _WIN32 - char fileSep = '\\'; - #else - char fileSep = '/'; - #endif - int status; - snprintf(tmp, sizeof(tmp),"%s",dir); - len = strlen(tmp); - if (tmp[len - 1] == fileSep) - tmp[len - 1] = 0; - for (p = tmp + 1; *p; p++){ - if (*p == fileSep) { - *p = 0; - - #ifdef _WIN32 - mkdir(tmp); - #else - mkdir(tmp, 0775); - #endif - - chmod(tmp, 0775); - *p = fileSep; - } - } - #ifdef _WIN32 - mkdir(tmp); - #else - mkdir(tmp, 0775); - #endif - chmod(tmp, 0775); -} - -void* mallocDynamic(uint64_t x, uint64_t bits){ - switch(bits){ - case 8: - return malloc(x*sizeof(uint8_t)); - case 16: - return malloc(x*sizeof(uint16_t)); - case 32: - return malloc(x*sizeof(float)); - case 64: - return malloc(x*sizeof(double)); - default: - printf("Image is not 8/16 bit, single, or double. Using single."); - return malloc(x*sizeof(float)); - } -} - -struct chunkAxisVals getChunkAxisVals(char* fileName){ - struct chunkAxisVals cAV; - char* ptr; - cAV.x = strtol(fileName, &ptr, 10); - ptr++; - cAV.y = strtol(ptr, &ptr, 10); - ptr++; - cAV.z = strtol(ptr, &ptr, 10); - return cAV; -} - -struct chunkInfo getChunkInfo(char* folderName, uint64_t startX, uint64_t startY, uint64_t startZ, uint64_t endX, uint64_t endY,uint64_t endZ,uint64_t chunkXSize,uint64_t chunkYSize,uint64_t chunkZSize){ - struct chunkInfo cI; - cI.numChunks = 0; - cI.chunkNames = NULL; - - uint64_t xStartAligned = startX-(startX%chunkXSize); - uint64_t yStartAligned = startY-(startY%chunkYSize); - uint64_t zStartAligned = startZ-(startZ%chunkZSize); - uint64_t xStartChunk = (xStartAligned/chunkXSize); - uint64_t yStartChunk = (yStartAligned/chunkYSize); - uint64_t zStartChunk = (zStartAligned/chunkZSize); - - uint64_t xEndAligned = endX; - uint64_t yEndAligned = endY; - uint64_t zEndAligned = endZ; - - if(xEndAligned%chunkXSize) xEndAligned = endX-(endX%chunkXSize)+chunkXSize; - if(yEndAligned%chunkYSize) yEndAligned = endY-(endY%chunkYSize)+chunkYSize; - if(zEndAligned%chunkZSize) zEndAligned = endZ-(endZ%chunkZSize)+chunkZSize; - uint64_t xEndChunk = (xEndAligned/chunkXSize); - uint64_t yEndChunk = (yEndAligned/chunkYSize); - uint64_t zEndChunk = (zEndAligned/chunkZSize); - - uint64_t xChunks = (xEndChunk-xStartChunk); - uint64_t yChunks = (yEndChunk-yStartChunk); - uint64_t zChunks = (zEndChunk-zStartChunk); - - uint64_t file_count = xChunks*yChunks*zChunks; - - char** chunkNames = malloc(file_count*sizeof(char*)); - #pragma omp parallel for collapse(3) - for(uint64_t x = xStartChunk; x < xEndChunk; x++){ - for(uint64_t y = yStartChunk; y < yEndChunk; y++){ - for(uint64_t z = zStartChunk; z < zEndChunk; z++){ - uint64_t currFile = (z-zStartChunk)+((y-yStartChunk)*zChunks)+((x-xStartChunk)*yChunks*zChunks); - asprintf(&chunkNames[currFile],"%llu.%llu.%llu",x,y,z); - } - } - } - cI.chunkNames = chunkNames; - cI.numChunks = file_count; - return cI; -} - -char* getSubfolderString(struct chunkAxisVals *cAV, uint64_t subfolderSizeX, uint64_t subfolderSizeY, uint64_t subfolderSizeZ){ - if(subfolderSizeX == 0 && subfolderSizeY == 0 && subfolderSizeZ == 0) return NULL; - - uint64_t currX = 0; - uint64_t currY = 0; - uint64_t currZ = 0; - if(subfolderSizeX > 0) currX = cAV->x/subfolderSizeX; - if(subfolderSizeY > 0) currY = cAV->y/subfolderSizeY; - if(subfolderSizeZ > 0) currZ = cAV->z/subfolderSizeZ; - - char* currName = NULL; - asprintf(&currName,"%llu_%llu_%llu",currX,currY,currZ); - return currName; - -} - -void setChunkShapeFromJSON(cJSON *json, uint64_t *x, uint64_t *y, uint64_t *z){ - *x = json->child->valueint; - *y = json->child->next->valueint; - *z = json->child->next->next->valueint; -} - -void setDTypeFromJSON(cJSON *json, char* dtype){ - dtype[0] = json->valuestring[0]; - dtype[1] = json->valuestring[1]; - dtype[2] = json->valuestring[2]; - dtype[3] = json->valuestring[3]; -} - -void setOrderFromJSON(cJSON *json, char* order){ - *order = json->valuestring[0]; -} - -void setShapeFromJSON(cJSON *json, uint64_t *x, uint64_t *y, uint64_t *z){ - *x = json->child->valueint; - *y = json->child->next->valueint; - *z = json->child->next->next->valueint; -} - -void setCnameFromJSON(cJSON *json, char** cname){ - cJSON *jsonItem = json->child; - - - while(jsonItem){ - if(!strcmp(jsonItem->string,"cname")){ - *cname = strdup(jsonItem->valuestring); - return; - } - // For gzip - if(!strcmp(jsonItem->string,"id") && !strcmp(jsonItem->valuestring,"gzip")){ - *cname = strdup(jsonItem->valuestring); - //mexErrMsgIdAndTxt("zarr:zarrayError","Compressor: \"%s\" is not currently supportedEXTRAIN\n",cname); - return; - } - jsonItem = jsonItem->next; - } - mexErrMsgIdAndTxt("zarr:zarrayError","Compressor: \"%s\" is not currently supported\n",*cname); - -} - -void setClevelFromJSON(cJSON *json, uint64_t* clevel){ - cJSON *jsonItem = json->child; - - while(jsonItem){ - if(!strcmp(jsonItem->string,"clevel")){ - *clevel = jsonItem->valueint; - return; - } - // For gzip - if(!strcmp(jsonItem->string,"level")){ - *clevel = jsonItem->valueint; - return; - } - jsonItem = jsonItem->next; - } - mexErrMsgIdAndTxt("zarr:zarrayError","Compression level not found in .zarray file\n"); - -} - -void setSubfoldersFromJSON(cJSON *json, uint64_t *subfolderSizeX, uint64_t *subfolderSizeY, uint64_t *subfolderSizeZ){ - *subfolderSizeX = json->child->valueint; - *subfolderSizeY = json->child->next->valueint; - *subfolderSizeZ = json->child->next->next->valueint; - -} - -void setValuesFromJSON(char* fileName,uint64_t *chunkXSize,uint64_t *chunkYSize,uint64_t *chunkZSize,char* dtype,char* order,uint64_t *shapeX,uint64_t *shapeY,uint64_t *shapeZ,char** cname, uint64_t* clevel, uint64_t *subfolderSizeX, uint64_t *subfolderSizeY, uint64_t *subfolderSizeZ){ - - char* zArray = ".zarray"; - char* fnFull = (char*)malloc(strlen(fileName)+9); - fnFull[0] = '\0'; - char fileSepS[2]; - fileSepS[0] = fileSep; - fileSepS[1] = '\0'; - - strcat(fnFull,fileName); - strcat(fnFull,fileSepS); - strcat(fnFull,zArray); - - FILE *fileptr = fopen(fnFull, "rb"); - if(!fileptr) mexErrMsgIdAndTxt("zarr:inputError","Failed to open JSON File: %s\n",fnFull); - free(fnFull); - - fseek(fileptr, 0, SEEK_END); - long filelen = ftell(fileptr); - rewind(fileptr); - char* buffer = (char *)malloc((filelen)); - fread(buffer, filelen, 1, fileptr); - fclose(fileptr); - cJSON *json = cJSON_ParseWithLength(buffer,filelen); - - while(json){ - if(!json->string){ - json = json->child; - continue; - } - else if(!strcmp(json->string,"chunks")){ - setChunkShapeFromJSON(json, chunkXSize,chunkYSize,chunkZSize); - } - else if(!strcmp(json->string,"dtype")){ - setDTypeFromJSON(json, dtype); - } - else if(!strcmp(json->string,"order")){ - setOrderFromJSON(json, order); - } - else if(!strcmp(json->string,"shape")){ - setShapeFromJSON(json, shapeX,shapeY,shapeZ); - } - else if(!strcmp(json->string,"compressor")){ - setCnameFromJSON(json, cname); - setClevelFromJSON(json, clevel); - } - else if(!strcmp(json->string,"subfolders")){ - setSubfoldersFromJSON(json, subfolderSizeX, subfolderSizeY, subfolderSizeZ); - } - json = json->next; - } - cJSON_Delete(json); -} - -void setJSONValues(char* fileName,uint64_t *chunkXSize,uint64_t *chunkYSize,uint64_t *chunkZSize,char* dtype,char* order,uint64_t *shapeX,uint64_t *shapeY,uint64_t *shapeZ, char* cname, uint64_t* clevel, uint64_t *subfolderSizeX, uint64_t *subfolderSizeY, uint64_t *subfolderSizeZ){ - - // Overflows for ints greater than 32 bit for chunkSizes and shape - cJSON* zArray = cJSON_CreateObject(); - const int chunkSizes[3] = {*chunkXSize,*chunkYSize,*chunkZSize}; - cJSON* chunks = cJSON_CreateIntArray(chunkSizes, 3); - cJSON_AddItemToObject(zArray, "chunks", chunks); - - cJSON* compressor = cJSON_CreateObject(); - cJSON_AddItemToObject(zArray, "compressor", compressor); - - if(!strcmp(cname,"lz4") || !strcmp(cname,"blosclz") || !strcmp(cname,"lz4hc") || !strcmp(cname,"zlib") || !strcmp(cname,"zstd")){ - cJSON_AddNumberToObject(compressor, "blocksize", 0); - cJSON_AddNumberToObject(compressor, "clevel", *clevel); - cJSON_AddStringToObject(compressor, "cname", cname); - cJSON_AddStringToObject(compressor, "id", "blosc"); - cJSON_AddNumberToObject(compressor, "shuffle", 1); - } - else if(!strcmp(cname,"gzip")){ - cJSON_AddStringToObject(compressor, "id", cname); - cJSON_AddNumberToObject(compressor, "level", *clevel); - } - else mexErrMsgIdAndTxt("zarr:zarrayError","Compressor: \"%s\" is not currently supported\n",cname); - - cJSON_AddStringToObject(zArray, "dtype", dtype); - cJSON_AddNumberToObject(zArray, "fill_value", 0); - cJSON_AddNullToObject(zArray, "filters"); - cJSON_AddStringToObject(zArray, "order", "F"); - - const int shapeSizes[3] = {*shapeX,*shapeY,*shapeZ}; - cJSON* shape = cJSON_CreateIntArray(shapeSizes, 3); - cJSON_AddItemToObject(zArray, "shape", shape); - cJSON_AddNumberToObject(zArray, "zarr_format", 2); - - const int subfolderSizeSizes[3] = {*subfolderSizeX,*subfolderSizeY,*subfolderSizeZ}; - cJSON* subfolderSize = cJSON_CreateIntArray(subfolderSizeSizes, 3); - cJSON_AddItemToObject(zArray, "subfolders", subfolderSize); - - uint64_t uuidLen; - #ifdef _WIN32 - uuidLen = 5; - char *uuid = malloc(uuidLen+1); - char *seedArr = malloc(1000); - struct timeval cSeed; - gettimeofday(&cSeed,NULL); - int nChars = sprintf(seedArr,"%d%d",cSeed.tv_sec,cSeed.tv_usec); - int aSeed = 0; - char* ptr; - if(nChars > 9) - aSeed = strtol(seedArr+nChars-10, &ptr, 9); - else aSeed = strtol(seedArr, &ptr, 9); - srand(aSeed); - sprintf(uuid,"%.5d",rand() % 99999); - free(seedArr); - #else - uuidLen = 36; - uuid_t binuuid; - uuid_generate_random(binuuid); - char *uuid = malloc(uuidLen+1); - uuid_unparse(binuuid, uuid); - #endif - - char* zArrayS = ".zarray"; - char* fnFull = (char*)malloc(strlen(fileName)+8+uuidLen+1); - fnFull[0] = '\0'; - char fileSepS[2]; - fileSepS[0] = fileSep; - fileSepS[1] = '\0'; - - strcat(fnFull,fileName); - strcat(fnFull,fileSepS); - strcat(fnFull,zArrayS); - char* fileNameFinal = strndup(fnFull,strlen(fileName)+1+8); - strcat(fnFull,uuid); - - char* string = cJSON_Print(zArray); - - FILE *fileptr = fopen(fnFull, "w+"); - if(!fileptr) mexErrMsgIdAndTxt("zarr:zarrayError","Cannot open %s\n",fnFull); - fprintf(fileptr,"%s",string); - fclose(fileptr); - - - rename(fnFull,fileNameFinal); - //file = fopen(fileNameFinal, "r"); - //if(!file) rename(fileName,fileNameFinal); - - cJSON_Delete(zArray); - free(fnFull); - free(uuid); - free(fileNameFinal); -} - -uint64_t fastCeilDiv(uint64_t num, uint64_t denom){ - return 1 + ((num - 1) / denom); -} - -void createSubfolders(char* folderName, uint64_t shapeX, uint64_t shapeY, uint64_t shapeZ, uint64_t chunkXSize, uint64_t chunkYSize, uint64_t chunkZSize, uint64_t subfolderSizeX, uint64_t subfolderSizeY, uint64_t subfolderSizeZ){ - if(subfolderSizeX == 0 && subfolderSizeY == 0 && subfolderSizeZ == 0) return; - uint64_t nChunksX = fastCeilDiv(shapeX,chunkXSize); - uint64_t nChunksY = fastCeilDiv(shapeY,chunkYSize); - uint64_t nChunksZ = fastCeilDiv(shapeZ,chunkZSize); - - - - - uint64_t nSubfoldersX = 1; - uint64_t nSubfoldersY = 1; - uint64_t nSubfoldersZ = 1; - - if(subfolderSizeX > 0) nSubfoldersX = fastCeilDiv(nChunksX,subfolderSizeX); - if(subfolderSizeY > 0) nSubfoldersY = fastCeilDiv(nChunksY,subfolderSizeY); - if(subfolderSizeZ > 0) nSubfoldersZ = fastCeilDiv(nChunksZ,subfolderSizeZ); - - //printf("%d,%d,%d\n",nSubfoldersX,nSubfoldersY,nSubfoldersZ); - //printf("%d,%d,%d\n",nChunksX,nChunksY,nChunksZ); - //printf("%d,%d,%d\n",subfolderSizeX,subfolderSizeY,subfolderSizeZ); - //mexErrMsgIdAndTxt("zarr:zarrayError","bork"); - #pragma omp parallel for collapse(3) - for(uint64_t x = 0; x < nSubfoldersX; x++){ - for(uint64_t y = 0; y < nSubfoldersY; y++){ - for(uint64_t z = 0; z < nSubfoldersZ; z++){ - char* currName = NULL; - asprintf(&currName,"%s/%llu_%llu_%llu",folderName,x,y,z); - mkdirRecursive(currName); - free(currName); - } - } - } -} diff --git a/parallelWriteZarr/helperFunctions.h b/parallelWriteZarr/helperFunctions.h deleted file mode 100755 index 26e735c..0000000 --- a/parallelWriteZarr/helperFunctions.h +++ /dev/null @@ -1,50 +0,0 @@ -#ifndef HELPERFUNCTIONS_H -#define HELPERFUNCTIONS_H -#include -#include - -#ifndef _WIN32 -#include -char* expandTilde(char* path); - -#endif - -#ifdef _WIN32 -char* strndup (const char *s, size_t n); - -#endif - -void* mallocDynamic(uint64_t x, uint64_t bits); - -struct chunkInfo{ - char** chunkNames; - int64_t numChunks; -}; - -struct chunkAxisVals{ - uint64_t x; - uint64_t y; - uint64_t z; -}; - -struct chunkAxisVals getChunkAxisVals(char* fileName); - -struct chunkInfo getChunkInfo(char* folderName, uint64_t startX, uint64_t startY, uint64_t startZ, uint64_t endX, uint64_t endY,uint64_t endZ,uint64_t chunkXSize,uint64_t chunkYSize,uint64_t chunkZSize); - -char* getSubfolderString(struct chunkAxisVals *cAV, uint64_t subfolderSizeX, uint64_t subfolderSizeY, uint64_t subfolderSizeZ); - -void setChunkShapeFromJSON(cJSON *json, uint64_t *x, uint64_t *y, uint64_t *z); - -void setDTypeFromJSON(cJSON *json, char* dtype); - -void setOrderFromJSON(cJSON *json, char* order); - -void setShapeFromJSON(cJSON *json, uint64_t *x, uint64_t *y, uint64_t *z); - -void setValuesFromJSON(char* fileName,uint64_t *chunkXSize,uint64_t *chunkYSize,uint64_t *chunkZSize,char* dtype,char* order,uint64_t *shapeX,uint64_t *shapeY,uint64_t *shapeZ,char** cname, uint64_t* clevel,uint64_t *subfolderSizeX,uint64_t *subfolderSizeY,uint64_t *subfolderSizeZ); - -void setJSONValues(char* fileName,uint64_t *chunkXSize,uint64_t *chunkYSize,uint64_t *chunkZSize,char* dtype,char* order,uint64_t *shapeX,uint64_t *shapeY,uint64_t *shapeZ, char* cname, uint64_t* clevel, uint64_t *subfolderSizeX, uint64_t *subfolderSizeY, uint64_t *subfolderSizeZ); - -void createSubfolders(char* folderName, uint64_t shapeX, uint64_t shapeY, uint64_t shapeZ, uint64_t chunkXSize, uint64_t chunkYSize, uint64_t chunkZSize, uint64_t subfolderSizeX, uint64_t subfolderSizeY, uint64_t subfolderSizeZ); - -#endif // HELPERFUNCTIONS_H diff --git a/parallelWriteZarr/linux/libblosc.so b/parallelWriteZarr/linux/libblosc.so deleted file mode 100755 index 9e54ed2..0000000 Binary files a/parallelWriteZarr/linux/libblosc.so and /dev/null differ diff --git a/parallelWriteZarr/linux/libblosc.so.1 b/parallelWriteZarr/linux/libblosc.so.1 deleted file mode 100755 index 9e54ed2..0000000 Binary files a/parallelWriteZarr/linux/libblosc.so.1 and /dev/null differ diff --git a/parallelWriteZarr/linux/libblosc.so.1.21.0 b/parallelWriteZarr/linux/libblosc.so.1.21.0 deleted file mode 100755 index 9e54ed2..0000000 Binary files a/parallelWriteZarr/linux/libblosc.so.1.21.0 and /dev/null differ diff --git a/parallelWriteZarr/linux/libblosc2.so b/parallelWriteZarr/linux/libblosc2.so deleted file mode 100755 index 308d3a7..0000000 Binary files a/parallelWriteZarr/linux/libblosc2.so and /dev/null differ diff --git a/parallelWriteZarr/linux/libblosc2.so.2 b/parallelWriteZarr/linux/libblosc2.so.2 deleted file mode 100755 index 308d3a7..0000000 Binary files a/parallelWriteZarr/linux/libblosc2.so.2 and /dev/null differ diff --git a/parallelWriteZarr/linux/libblosc2.so.2.0.5 b/parallelWriteZarr/linux/libblosc2.so.2.0.5 deleted file mode 100755 index 308d3a7..0000000 Binary files a/parallelWriteZarr/linux/libblosc2.so.2.0.5 and /dev/null differ diff --git a/parallelWriteZarr/linux/libcjson.so b/parallelWriteZarr/linux/libcjson.so deleted file mode 100755 index eb168c2..0000000 Binary files a/parallelWriteZarr/linux/libcjson.so and /dev/null differ diff --git a/parallelWriteZarr/linux/libcjson.so.1 b/parallelWriteZarr/linux/libcjson.so.1 deleted file mode 100755 index eb168c2..0000000 Binary files a/parallelWriteZarr/linux/libcjson.so.1 and /dev/null differ diff --git a/parallelWriteZarr/linux/libcjson.so.1.7.15 b/parallelWriteZarr/linux/libcjson.so.1.7.15 deleted file mode 100755 index eb168c2..0000000 Binary files a/parallelWriteZarr/linux/libcjson.so.1.7.15 and /dev/null differ diff --git a/parallelWriteZarr/linux/libgomp.so b/parallelWriteZarr/linux/libgomp.so deleted file mode 100755 index eccfabc..0000000 Binary files a/parallelWriteZarr/linux/libgomp.so and /dev/null differ diff --git a/parallelWriteZarr/linux/libgomp.so.1 b/parallelWriteZarr/linux/libgomp.so.1 deleted file mode 100755 index eccfabc..0000000 Binary files a/parallelWriteZarr/linux/libgomp.so.1 and /dev/null differ diff --git a/parallelWriteZarr/linux/libgomp.so.1.0.0 b/parallelWriteZarr/linux/libgomp.so.1.0.0 deleted file mode 100755 index eccfabc..0000000 Binary files a/parallelWriteZarr/linux/libgomp.so.1.0.0 and /dev/null differ diff --git a/parallelWriteZarr/linux/libz.so b/parallelWriteZarr/linux/libz.so deleted file mode 100755 index b377c07..0000000 Binary files a/parallelWriteZarr/linux/libz.so and /dev/null differ diff --git a/parallelWriteZarr/linux/libz.so.1 b/parallelWriteZarr/linux/libz.so.1 deleted file mode 100755 index b377c07..0000000 Binary files a/parallelWriteZarr/linux/libz.so.1 and /dev/null differ diff --git a/parallelWriteZarr/linux/libz.so.1.2.11 b/parallelWriteZarr/linux/libz.so.1.2.11 deleted file mode 100755 index b377c07..0000000 Binary files a/parallelWriteZarr/linux/libz.so.1.2.11 and /dev/null differ diff --git a/parallelWriteZarr/linux/parallelWriteZarr.mexa64 b/parallelWriteZarr/linux/parallelWriteZarr.mexa64 deleted file mode 100755 index e49d1b9..0000000 Binary files a/parallelWriteZarr/linux/parallelWriteZarr.mexa64 and /dev/null differ diff --git a/parallelWriteZarr/parallelReadZarr.c b/parallelWriteZarr/parallelReadZarr.c deleted file mode 100755 index aa69aac..0000000 --- a/parallelWriteZarr/parallelReadZarr.c +++ /dev/null @@ -1,312 +0,0 @@ -#include "parallelReadZarr.h" -#include -#include -#include -#include -#include -#include -#include "helperFunctions.h" - -#include "zlib.h" - -//mex -v COPTIMFLAGS="-O3 -DNDEBUG" CFLAGS='$CFLAGS -O3 -fopenmp' LDFLAGS='$LDFLAGS -O3 -fopenmp' '-I/global/home/groups/software/sl-7.x86_64/modules/cBlosc/2.0.4/include/' '-I/global/home/groups/software/sl-7.x86_64/modules/cJSON/1.7.15/include/' '-L/global/home/groups/software/sl-7.x86_64/modules/cBlosc/2.0.4/lib64' -lblosc2 '-L/global/home/groups/software/sl-7.x86_64/modules/cJSON/1.7.15/lib64' -lcjson zarrMex.c - -void parallelReadZarr(void* zarr, char* folderName, char* subfolderName,uint64_t startX, uint64_t startY, uint64_t startZ, uint64_t endX, uint64_t endY,uint64_t endZ,uint64_t chunkXSize,uint64_t chunkYSize,uint64_t chunkZSize,uint64_t shapeX,uint64_t shapeY,uint64_t shapeZ, uint64_t bits, char order, char* cname){ - char fileSepS[2]; - const char fileSep = -#ifdef _WIN32 - '\\'; -#else - '/'; -#endif - - uint64_t bytes = (bits/8); - fileSepS[0] = fileSep; - fileSepS[1] = '\0'; - - /* Initialize the Blosc compressor */ - int32_t numWorkers = omp_get_max_threads(); - - struct chunkInfo cI = getChunkInfo(folderName, startX, startY, startZ, endX, endY, endZ,chunkXSize,chunkYSize,chunkZSize); - //if(!cI.chunkNames) mexErrMsgIdAndTxt("zarr:inputError","File \"%s\" cannot be opened",folderName); - - int32_t batchSize = (cI.numChunks-1)/numWorkers+1; - uint64_t s = chunkXSize*chunkYSize*chunkZSize; - uint64_t sB = s*bytes; - int32_t w; - int err = 0; - char errString[10000]; - -#pragma omp parallel for //if(numWorkers<=cI.numChunks) - for(w = 0; w < numWorkers; w++){ - void* bufferDest = mallocDynamic(s,bits); - uint64_t lastFileLen = 0; - char *buffer = NULL; - for(int64_t f = w*batchSize; f < (w+1)*batchSize; f++){ - if(f>=cI.numChunks || err) break; - struct chunkAxisVals cAV = getChunkAxisVals(cI.chunkNames[f]); - - //malloc +2 for null term and filesep - char *fileName = malloc(strlen(folderName)+1+strlen(subfolderName)+1+strlen(cI.chunkNames[f])+1); - fileName[0] = '\0'; - strcat(fileName,folderName); - strcat(fileName,fileSepS); - strcat(fileName,subfolderName); - strcat(fileName,fileSepS); - strcat(fileName,cI.chunkNames[f]); - - FILE *fileptr = fopen(fileName, "rb"); - if(!fileptr){ - #pragma omp critical - { - memset(zarr,0,sB); - free(fileName); - free(subfolderName); - //err = 1; - //sprintf(errString,"Could not open file: %s\n",fileName); - } - break; - } - free(fileName); - free(subfolderName); - - fseek(fileptr, 0, SEEK_END); - long filelen = ftell(fileptr); - rewind(fileptr); - if(lastFileLen < filelen){ - free(buffer); - buffer = (char*) malloc(filelen); - lastFileLen = filelen; - } - fread(buffer, filelen, 1, fileptr); - fclose(fileptr); - - // Decompress - int dsize = -1; - int uncErr = 0; - if(strcmp(cname,"gzip")){ - //dsize = blosc2_decompress(buffer, filelen, bufferDest, sB); - blosc2_context *dctx; - blosc2_dparams dparams = BLOSC2_DPARAMS_DEFAULTS; - dctx = blosc2_create_dctx(dparams); - - dsize = blosc2_decompress_ctx(dctx, buffer, filelen,bufferDest, sB); - blosc2_free_ctx(dctx); - } - else{ - dsize = sB; - z_stream stream; - stream.zalloc = Z_NULL; - stream.zfree = Z_NULL; - stream.opaque = Z_NULL; - stream.avail_in = (uInt)filelen; - stream.avail_out = (uInt)sB; - while(stream.avail_in > 0){ - - dsize = sB; - - stream.next_in = (uint8_t*)buffer+(filelen-stream.avail_in); - stream.next_out = (uint8_t*)bufferDest+(sB-stream.avail_out); - - uncErr = inflateInit2(&stream, 32); - if(uncErr){ - #pragma omp critical - { - err = 1; - sprintf(errString,"Decompression error. Error code: %d ChunkName: %s/%s\n",uncErr,folderName,cI.chunkNames[f]); - } - break; - } - - uncErr = inflate(&stream, Z_NO_FLUSH); - - if(uncErr != Z_STREAM_END){ - #pragma omp critical - { - err = 1; - sprintf(errString,"Decompression error. Error code: %d ChunkName: %s/%s\n",uncErr,folderName,cI.chunkNames[f]); - } - break; - } - } - if(inflateEnd(&stream)){ - #pragma omp critical - { - err = 1; - sprintf(errString,"Decompression error. Error code: %d ChunkName: %s/%s\n",uncErr,folderName,cI.chunkNames[f]); - } - break; - } - } - - - if(dsize < 0){ -#pragma omp critical - { - err = 1; - sprintf(errString,"Decompression error. Error code: %d ChunkName: %s/%s\n",dsize,folderName,cI.chunkNames[f]); - } - break; - } - - //printf("ChunkName: %s\n",cI.chunkNames[f]); - //printf("w: %d b: %d\n",w,f); - if(order == 'F'){ - for(int64_t z = cAV.z*chunkZSize; z < (cAV.z+1)*chunkZSize; z++){ - if(z>=endZ) break; - else if(z=endY) break; - else if(y startX) || (cAV.x+1)*chunkXSize>endX){ - if(((cAV.x*chunkXSize) < startX && ((cAV.x+1)*chunkXSize) > startX) && (cAV.x+1)*chunkXSize>endX){ - memcpy((uint8_t*)zarr+((((cAV.x*chunkXSize)-startX+(startX%chunkXSize))+((y-startY)*shapeX)+((z-startZ)*shapeX*shapeY))*bytes),(uint8_t*)bufferDest+(((startX%chunkXSize)+((y%chunkYSize)*chunkXSize)+((z%chunkZSize)*chunkXSize*chunkYSize))*bytes),((endX%chunkXSize)-(startX%chunkXSize))*bytes); - } - else if((cAV.x+1)*chunkXSize>endX){ - memcpy((uint8_t*)zarr+((((cAV.x*chunkXSize)-startX)+((y-startY)*shapeX)+((z-startZ)*shapeX*shapeY))*bytes),(uint8_t*)bufferDest+((((y%chunkYSize)*chunkXSize)+((z%chunkZSize)*chunkXSize*chunkYSize))*bytes),(endX%chunkXSize)*bytes); - } - else if((cAV.x*chunkXSize) < startX && ((cAV.x+1)*chunkXSize) > startX){ - memcpy((uint8_t*)zarr+((((cAV.x*chunkXSize-startX+(startX%chunkXSize)))+((y-startY)*shapeX)+((z-startZ)*shapeX*shapeY))*bytes),(uint8_t*)bufferDest+(((startX%chunkXSize)+((y%chunkYSize)*chunkXSize)+((z%chunkZSize)*chunkXSize*chunkYSize))*bytes),(chunkXSize-(startX%chunkXSize))*bytes); - } - } - else{ - memcpy((uint8_t*)zarr+((((cAV.x*chunkXSize)-startX)+((y-startY)*shapeX)+((z-startZ)*shapeX*shapeY))*bytes),(uint8_t*)bufferDest+((((y%chunkYSize)*chunkXSize)+((z%chunkZSize)*chunkXSize*chunkYSize))*bytes),chunkXSize*bytes); - } - } - } - } - else if (order == 'C'){ - for(int64_t x = cAV.x*chunkXSize; x < (cAV.x+1)*chunkXSize; x++){ - if(x>=endX) break; - else if(x=endY) break; - else if(y=endZ) break; - else if(z shapeX || endY > shapeY || endZ > shapeZ){ - * printf("Upper bound is invalid\n"); - * return NULL; - * }*/ - uint64_t dim[3]; - shapeX = endX-startX; - shapeY = endY-startY; - shapeZ = endZ-startZ; - dim[0] = shapeX; - dim[1] = shapeY; - dim[2] = shapeZ; - - if(dtype[1] == 'u' && dtype[2] == '1'){ - uint64_t bits = 8; - uint8_t* zarr = (uint8_t*)malloc(sizeof(uint8_t)*shapeX*shapeY*shapeZ); - parallelReadZarr((void*)zarr,folderName,subfolderName,startX,startY,startZ,endX,endY,endZ,chunkXSize,chunkYSize,chunkZSize,shapeX,shapeY,shapeZ,bits,order,cname); - return (void*)zarr; - } - else if(dtype[1] == 'u' && dtype[2] == '2'){ - uint64_t bits = 16; - uint16_t* zarr = (uint16_t*)malloc((uint64_t)(sizeof(uint16_t)*shapeX*shapeY*shapeZ)); - parallelReadZarr((void*)zarr,folderName,subfolderName,startX,startY,startZ,endX,endY,endZ,chunkXSize,chunkYSize,chunkZSize,shapeX,shapeY,shapeZ,bits,order,cname); - return (void*)zarr; - } - else if(dtype[1] == 'f' && dtype[2] == '4'){ - uint64_t bits = 32; - float* zarr = (float*)malloc(sizeof(float)*shapeX*shapeY*shapeZ); - parallelReadZarr((void*)zarr,folderName,subfolderName,startX,startY,startZ,endX,endY,endZ,chunkXSize,chunkYSize,chunkZSize,shapeX,shapeY,shapeZ,bits,order,cname); - return (void*)zarr; - } - else if(dtype[1] == 'f' && dtype[2] == '8'){ - uint64_t bits = 64; - double* zarr = (double*)malloc(sizeof(double)*shapeX*shapeY*shapeZ); - parallelReadZarr((void*)zarr,folderName,subfolderName,startX,startY,startZ,endX,endY,endZ,chunkXSize,chunkYSize,chunkZSize,shapeX,shapeY,shapeZ,bits,order,cname); - return (void*)zarr; - } - else{ - return NULL; - } -} diff --git a/parallelWriteZarr/parallelReadZarr.h b/parallelWriteZarr/parallelReadZarr.h deleted file mode 100755 index 53bf830..0000000 --- a/parallelWriteZarr/parallelReadZarr.h +++ /dev/null @@ -1,9 +0,0 @@ -#ifndef PARALLELREADZARR_H -#define PARALLELREADZARR_H -#include - -uint64_t dTypeToBits(char* dtype); - -void* parallelReadZarrWrapper(char* folderName,uint8_t crop, char* subfolderName, uint64_t startX, uint64_t startY, uint64_t startZ, uint64_t endX, uint64_t endY,uint64_t endZ); - -#endif // PARALLELREADZARR_H diff --git a/parallelWriteZarr/parallelWriteZarr.c b/parallelWriteZarr/parallelWriteZarr.c deleted file mode 100755 index d574a99..0000000 --- a/parallelWriteZarr/parallelWriteZarr.c +++ /dev/null @@ -1,708 +0,0 @@ -#include "parallelWriteZarr.h" -#include "parallelReadZarr.h" -#include "helperFunctions.h" -#include -#include -#include -#include -#include -#include -#include -#include -#ifdef _WIN32 -#include -#else -#include -#endif -#include -#include "mex.h" - -#include "zlib.h" - -//compile -//mex -v COPTIMFLAGS="-DNDEBUG -O3" CFLAGS='$CFLAGS -fopenmp -O3' LDFLAGS='$LDFLAGS -fopenmp -O3' '-I/global/home/groups/software/sl-7.x86_64/modules/cBlosc/2.0.4/include/' '-I/global/home/groups/software/sl-7.x86_64/modules/cBlosc/zarr/include/' '-I/global/home/groups/software/sl-7.x86_64/modules/cJSON/1.7.15/include/' '-L/global/home/groups/software/sl-7.x86_64/modules/cBlosc/zarr/lib' -lblosc '-L/global/home/groups/software/sl-7.x86_64/modules/cBlosc/2.0.4/lib64' -lblosc2 '-L/global/home/groups/software/sl-7.x86_64/modules/cJSON/1.7.15/lib64' -lcjson -luuid parallelWriteZarr.c helperFunctions.c parallelReadZarr.c - -//With zlib -//mex -v COPTIMFLAGS="-DNDEBUG -O3" LDOPTIMFLAGS="-Wl',-rpath='''$ORIGIN'''' -O3 -DNDEBUG" CFLAGS='$CFLAGS -fopenmp -O3' LDFLAGS='$LDFLAGS -fopenmp -O3' '-I/global/home/groups/software/sl-7.x86_64/modules/cBlosc/2.0.4/include/' '-I/global/home/groups/software/sl-7.x86_64/modules/cBlosc/zarr/include/' '-I/global/home/groups/software/sl-7.x86_64/modules/cJSON/1.7.15/include/' '-L/global/home/groups/software/sl-7.x86_64/modules/cBlosc/zarr/lib' -lblosc '-L/global/home/groups/software/sl-7.x86_64/modules/cBlosc/2.0.4/lib64' -lblosc2 '-L/global/home/groups/software/sl-7.x86_64/modules/cJSON/1.7.15/lib64' -lcjson -luuid -lz parallelWriteZarr.c helperFunctions.c parallelReadZarr.c - -//mex -v COPTIMFLAGS="-O3 -fwrapv -DNDEBUG" CFLAGS='$CFLAGS -O3 -fopenmp' LDFLAGS='$LDFLAGS -O3 -fopenmp' '-I/global/home/groups/software/sl-7.x86_64/modules/cBlosc/2.0.4/include/' '-L/global/home/groups/software/sl-7.x86_64/modules/cBlosc/2.0.4/lib64' -lblosc2 zarrMex.c -// -//Windows -//mex -v COPTIMFLAGS="-O3 -DNDEBUG" CFLAGS='$CFLAGS -O3 -fopenmp' LDFLAGS='$LDFLAGS -O3 -fopenmp' '-IC:\Program Files (x86)\bloscZarr\include' '-LC:\Program Files (x86)\bloscZarr\lib' -lblosc '-IC:\Program Files (x86)\cJSON\include\' '-LC:\Program Files (x86)\cJSON\lib' -lcjson '-IC:\Program Files (x86)\blosc\include' '-LC:\Program Files (x86)\blosc\lib' -lblosc2 parallelWriteZarr.c parallelReadZarr.c helperFunctions.c -void parallelWriteZarrMex(void* zarr, char* folderName,uint64_t startX, uint64_t startY, uint64_t startZ, uint64_t endX, uint64_t endY,uint64_t endZ,uint64_t chunkXSize,uint64_t chunkYSize,uint64_t chunkZSize,uint64_t shapeX,uint64_t shapeY,uint64_t shapeZ,uint64_t origShapeX,uint64_t origShapeY,uint64_t origShapeZ, uint64_t bits, char order, uint8_t useUuid, uint8_t crop, char* cname, uint64_t clevel, uint64_t subfolderSizeX, uint64_t subfolderSizeY, uint64_t subfolderSizeZ){ - char fileSepS[2]; - const char fileSep = - #ifdef _WIN32 - '\\'; - #else - '/'; - #endif - fileSepS[0] = fileSep; - fileSepS[1] = '\0'; - - //printf("%s startxyz: %d %d %d endxyz: %d %d %d chunkxyz: %d %d %d shapexyz: %d %d %d bits: %d\n",folderName,startX,startY,startZ,endX,endY,endZ,chunkXSize,chunkYSize,chunkZSize,shapeX,shapeY,shapeZ,bits); - - - uint64_t bytes = (bits/8); - - /* Initialize the Blosc compressor */ - int32_t numWorkers = omp_get_max_threads(); - - struct chunkInfo cI = getChunkInfo(folderName,startX,startY,startZ,endX,endY,endZ,chunkXSize,chunkYSize,chunkZSize); - //if(!cI.chunkNames) mexErrMsgIdAndTxt("zarr:inputError","File \"%s\" cannot be opened",folderName); - - int32_t batchSize = (cI.numChunks-1)/numWorkers+1; - uint64_t s = chunkXSize*chunkYSize*chunkZSize; - uint64_t sB = s*bytes; - int32_t w; - - uint64_t xRest = 0; - uint64_t yRest = 0; - uint64_t zRest = 0; - - uint64_t uuidLen; - #ifdef _WIN32 - uuidLen = 5; - char *uuid = malloc(uuidLen+1); - char *seedArr = malloc(1000); - struct timeval cSeed; - gettimeofday(&cSeed,NULL); - int nChars = sprintf(seedArr,"%d%d",cSeed.tv_sec,cSeed.tv_usec); - int aSeed = 0; - char* ptr; - if(nChars > 9) - aSeed = strtol(seedArr+nChars-10, &ptr, 9); - else aSeed = strtol(seedArr, &ptr, 9); - srand(aSeed); - sprintf(uuid,"%.5d",rand() % 99999); - free(seedArr); - #else - uuidLen = 36; - uuid_t binuuid; - uuid_generate_random(binuuid); - char *uuid = malloc(uuidLen+1); - uuid_unparse(binuuid, uuid); - #endif - int err = 0; - char errString[10000]; - #pragma omp parallel for if(numWorkers<=cI.numChunks) - for(w = 0; w < numWorkers; w++){ - void* chunkUnC = mallocDynamic(s,bits); - void* chunkC = malloc(sB+BLOSC_MAX_OVERHEAD); - for(int64_t f = w*batchSize; f < (w+1)*batchSize; f++){ - if(f>=cI.numChunks || err) break; - struct chunkAxisVals cAV = getChunkAxisVals(cI.chunkNames[f]); - void* cRegion = NULL; - char* subfolderName = getSubfolderString(&cAV,subfolderSizeX,subfolderSizeY,subfolderSizeZ); - if(!subfolderName){ - subfolderName = malloc(1); - subfolderName[0] = '\0'; - } - if(crop && ((((cAV.x)*chunkXSize) < startX || ((cAV.x+1)*chunkXSize > endX && endX < origShapeX)) - || (((cAV.y)*chunkYSize) < startY || ((cAV.y+1)*chunkYSize > endY && endY < origShapeY)) - || (((cAV.z)*chunkZSize) < startZ || ((cAV.z+1)*chunkZSize > endZ && endZ < origShapeZ)))){ - cRegion = parallelReadZarrWrapper(folderName, crop, subfolderName, ((cAV.x)*chunkXSize)+1, ((cAV.y)*chunkYSize)+1, ((cAV.z)*chunkZSize)+1, (cAV.x+1)*chunkXSize, (cAV.y+1)*chunkYSize, (cAV.z+1)*chunkZSize); - } - if(order == 'F'){ - for(int64_t z = cAV.z*chunkZSize; z < (cAV.z+1)*chunkZSize; z++){ - if(z>=endZ){ - if(crop){ - if((cAV.z+1)*chunkZSize > origShapeZ){ - memcpy((uint8_t*)chunkUnC+((((z%chunkZSize)*chunkXSize*chunkYSize))*bytes),(uint8_t*)cRegion+((((z%chunkZSize)*chunkXSize*chunkYSize))*bytes),((origShapeZ-z)*chunkXSize*chunkYSize)*bytes); - uint64_t zRest = ((cAV.z+1)*chunkZSize)-origShapeZ; - memset((uint8_t*)chunkUnC+(((z%chunkZSize)*chunkXSize*chunkYSize)*bytes),0,(zRest*(chunkXSize*chunkYSize))*bytes); - } - else{ - memcpy((uint8_t*)chunkUnC+((((z%chunkZSize)*chunkXSize*chunkYSize))*bytes),(uint8_t*)cRegion+((((z%chunkZSize)*chunkXSize*chunkYSize))*bytes),((((cAV.z+1)*chunkZSize)-z)*chunkXSize*chunkYSize)*bytes); - } - } - else{ - uint64_t zRest = ((cAV.z+1)*chunkZSize)-z; - memset((uint8_t*)chunkUnC+(((z%chunkZSize)*chunkXSize*chunkYSize)*bytes),0,(zRest*(chunkXSize*chunkYSize))*bytes); - } - break; - } - else if(z=endY){ - if(crop){ - if((cAV.y+1)*chunkYSize > origShapeY){ - memcpy((uint8_t*)chunkUnC+((((y%chunkYSize)*chunkXSize)+((z%chunkZSize)*chunkXSize*chunkYSize))*bytes),(uint8_t*)cRegion+((((y%chunkYSize)*chunkXSize)+((z%chunkZSize)*chunkXSize*chunkYSize))*bytes),((origShapeY-y)*chunkXSize)*bytes); - uint64_t yRest = ((cAV.y+1)*chunkYSize)-origShapeY; - memset((uint8_t*)chunkUnC+((((y%chunkYSize)*chunkXSize)+((z%chunkZSize)*chunkXSize*chunkYSize))*bytes),0,(yRest*(chunkXSize))*bytes); - } - else{ - memcpy((uint8_t*)chunkUnC+((((y%chunkYSize)*chunkXSize)+((z%chunkZSize)*chunkXSize*chunkYSize))*bytes),(uint8_t*)cRegion+((((y%chunkYSize)*chunkXSize)+((z%chunkZSize)*chunkXSize*chunkYSize))*bytes),((((cAV.y+1)*chunkYSize)-y)*chunkXSize)*bytes); - } - } - else{ - uint64_t yRest = ((cAV.y+1)*chunkYSize)-y; - memset((uint8_t*)chunkUnC+((((y%chunkYSize)*chunkXSize)+((z%chunkZSize)*chunkXSize*chunkYSize))*bytes),0,(yRest*chunkXSize)*bytes); - } - break; - } - else if(y startX) || (cAV.x+1)*chunkXSize>endX){ - if(((cAV.x*chunkXSize) < startX && ((cAV.x+1)*chunkXSize) > startX) && (cAV.x+1)*chunkXSize>endX){ - if(crop){ - memcpy((uint8_t*)chunkUnC+((((y%chunkYSize)*chunkXSize)+((z%chunkZSize)*chunkXSize*chunkYSize))*bytes),(uint8_t*)cRegion+((((y%chunkYSize)*chunkXSize)+((z%chunkZSize)*chunkXSize*chunkYSize))*bytes),(startX%chunkXSize)*bytes); - memcpy((uint8_t*)chunkUnC+(((startX%chunkXSize)+((y%chunkYSize)*chunkXSize)+((z%chunkZSize)*chunkXSize*chunkYSize))*bytes),(uint8_t*)zarr+((((cAV.x*chunkXSize)-startX+(startX%chunkXSize))+((y-startY)*shapeX)+((z-startZ)*shapeX*shapeY))*bytes),((endX%chunkXSize)-(startX%chunkXSize))*bytes); - memcpy((uint8_t*)chunkUnC+(((((y%chunkYSize)*chunkXSize)+((z%chunkZSize)*chunkXSize*chunkYSize))+(endX%chunkXSize))*bytes),(uint8_t*)cRegion+(((((y%chunkYSize)*chunkXSize)+((z%chunkZSize)*chunkXSize*chunkYSize))+(endX%chunkXSize))*bytes),(chunkXSize-(endX%chunkXSize))*bytes); - } - else{ - memset((uint8_t*)chunkUnC+((((y%chunkYSize)*chunkXSize)+((z%chunkZSize)*chunkXSize*chunkYSize))*bytes),0,(startX%chunkXSize)*bytes); - memcpy((uint8_t*)chunkUnC+(((startX%chunkXSize)+((y%chunkYSize)*chunkXSize)+((z%chunkZSize)*chunkXSize*chunkYSize))*bytes),(uint8_t*)zarr+((((cAV.x*chunkXSize)-startX+(startX%chunkXSize))+((y-startY)*shapeX)+((z-startZ)*shapeX*shapeY))*bytes),((endX%chunkXSize)-(startX%chunkXSize))*bytes); - memset((uint8_t*)chunkUnC+((((y%chunkYSize)*chunkXSize)+((z%chunkZSize)*chunkXSize*chunkYSize))+(endX%chunkXSize)*bytes),0,(chunkXSize-(endX%chunkXSize))*bytes); - } - } - else if((cAV.x+1)*chunkXSize>endX){ - if(crop){ - memcpy((uint8_t*)chunkUnC+((((y%chunkYSize)*chunkXSize)+((z%chunkZSize)*chunkXSize*chunkYSize))*bytes),(uint8_t*)zarr+((((cAV.x*chunkXSize)-startX)+((y-startY)*shapeX)+((z-startZ)*shapeX*shapeY))*bytes),(endX-(cAV.x*chunkXSize))*bytes); - - if((cAV.x+1)*chunkXSize > origShapeX){ - memcpy((uint8_t*)chunkUnC+((((endX-(cAV.x*chunkXSize)))+((y%chunkYSize)*chunkXSize)+((z%chunkZSize)*chunkXSize*chunkYSize))*bytes),(uint8_t*)cRegion+((((endX-(cAV.x*chunkXSize)))+((y%chunkYSize)*chunkXSize)+((z%chunkZSize)*chunkXSize*chunkYSize))*bytes),(origShapeX-endX)*bytes); - uint64_t xRest = ((cAV.x+1)*chunkXSize)-origShapeX; - memset((uint8_t*)chunkUnC+(((origShapeX-(cAV.x*chunkXSize))+((y%chunkYSize)*chunkXSize)+((z%chunkZSize)*chunkXSize*chunkYSize))*bytes),0,(xRest)*bytes); - } - else{ - memcpy((uint8_t*)chunkUnC+((((endX-(cAV.x*chunkXSize)))+((y%chunkYSize)*chunkXSize)+((z%chunkZSize)*chunkXSize*chunkYSize))*bytes),(uint8_t*)cRegion+((((endX-(cAV.x*chunkXSize)))+((y%chunkYSize)*chunkXSize)+((z%chunkZSize)*chunkXSize*chunkYSize))*bytes),(((cAV.x+1)*chunkXSize)-endX)*bytes); - } - } - else{ - memcpy((uint8_t*)chunkUnC+((((y%chunkYSize)*chunkXSize)+((z%chunkZSize)*chunkXSize*chunkYSize))*bytes),(uint8_t*)zarr+((((cAV.x*chunkXSize)-startX)+((y-startY)*shapeX)+((z-startZ)*shapeX*shapeY))*bytes),(endX%chunkXSize)*bytes); - memset((uint8_t*)chunkUnC+(((((y%chunkYSize)*chunkXSize)+((z%chunkZSize)*chunkXSize*chunkYSize))+(endX%chunkXSize))*bytes),0,(chunkXSize-(endX%chunkXSize))*bytes); - } - } - else if((cAV.x*chunkXSize) < startX && ((cAV.x+1)*chunkXSize) > startX){ - if(crop){ - memcpy((uint8_t*)chunkUnC+((((y%chunkYSize)*chunkXSize)+((z%chunkZSize)*chunkXSize*chunkYSize))*bytes),(uint8_t*)cRegion+((((y%chunkYSize)*chunkXSize)+((z%chunkZSize)*chunkXSize*chunkYSize))*bytes),(startX%chunkXSize)*bytes); - memcpy((uint8_t*)chunkUnC+(((startX%chunkXSize)+((y%chunkYSize)*chunkXSize)+((z%chunkZSize)*chunkXSize*chunkYSize))*bytes),(uint8_t*)zarr+((((cAV.x*chunkXSize)-startX+(startX%chunkXSize))+((y-startY)*shapeX)+((z-startZ)*shapeX*shapeY))*bytes),(chunkXSize-(startX%chunkXSize))*bytes); - } - else{ - memset((uint8_t*)chunkUnC+((((y%chunkYSize)*chunkXSize)+((z%chunkZSize)*chunkXSize*chunkYSize))*bytes),0,(startX%chunkXSize)*bytes); - memcpy((uint8_t*)chunkUnC+(((startX%chunkXSize)+((y%chunkYSize)*chunkXSize)+((z%chunkZSize)*chunkXSize*chunkYSize))*bytes),(uint8_t*)zarr+((((cAV.x*chunkXSize)-startX+(startX%chunkXSize))+((y-startY)*shapeX)+((z-startZ)*shapeX*shapeY))*bytes),(chunkXSize-(startX%chunkXSize))*bytes); - } - } - } - else{ - memcpy((uint8_t*)chunkUnC+((((y%chunkYSize)*chunkXSize)+((z%chunkZSize)*chunkXSize*chunkYSize))*bytes),(uint8_t*)zarr+((((cAV.x*chunkXSize)-startX)+((y-startY)*shapeX)+((z-startZ)*shapeX*shapeY))*bytes),chunkXSize*bytes); - } - } - } - } - else if (order == 'C'){ - for(int64_t x = cAV.x*chunkZSize; x < (cAV.x+1)*chunkXSize; x++){ - for(int64_t y = cAV.y*chunkYSize; y < (cAV.y+1)*chunkYSize; y++){ - for(int64_t z = cAV.z*chunkZSize; z < (cAV.z+1)*chunkZSize; z++){ - switch(bytes){ - case 1: - if(x>=endX || x= endY || y=endZ || z=endX || x= endY || y=endZ || z=endX || x= endY || y=endZ || z=endX || x= endY || y=endZ || z iDims[0] || endY-startY > iDims[1] || endZ-startZ > iDims[2]) mexErrMsgIdAndTxt("zarr:inputError","Bounds are invalid for the input data size"); - - if(nrhs >= 5){ - cname = mxArrayToString(prhs[4]); - } - if(nrhs == 6){ - if(mxGetN(prhs[5]) != 3) mexErrMsgIdAndTxt("zarr:inputError","subfolders must be an array of 3 numbers\n"); - subfolderSizeX = (uint64_t)*(mxGetPr(prhs[5])); - subfolderSizeY = (uint64_t)*((mxGetPr(prhs[5])+1)); - subfolderSizeZ = (uint64_t)*((mxGetPr(prhs[5])+2)); - } - } - else if(mxGetN(prhs[3]) != 3) mexErrMsgIdAndTxt("zarr:inputError","Input range is not 6 or 3"); - } - else if (nrhs > 6) mexErrMsgIdAndTxt("zarr:inputError","Number of input arguments must be 4 or less"); - if(!mxIsChar(prhs[0])) mexErrMsgIdAndTxt("zarr:inputError","The first argument must be a string"); - char* folderName = mxArrayToString(prhs[0]); - // Handle the tilde character in filenames on Linux/Mac - #ifndef _WIN32 - if(strchr(folderName,'~')) folderName = expandTilde(folderName); - #endif - uint8_t useUuid = (uint8_t)*(mxGetPr(prhs[2])); - uint64_t shapeX = 0; - uint64_t shapeY = 0; - uint64_t shapeZ = 0; - uint64_t chunkXSize = 0; - uint64_t chunkYSize = 0; - uint64_t chunkZSize = 0; - char dtype[4]; - char order; - void* zarrC = NULL; - - - mxClassID mDType = mxGetClassID(prhs[1]); - dtype[0] = '<'; - if(mDType == mxUINT8_CLASS){ - dtype[1] = 'u'; - dtype[2] = '1'; - - } - else if(mDType == mxUINT16_CLASS){ - dtype[1] = 'u'; - dtype[2] = '2'; - - } - else if(mDType == mxSINGLE_CLASS){ - dtype[1] = 'f'; - dtype[2] = '4'; - - } - else if(mDType == mxDOUBLE_CLASS){ - dtype[1] = 'f'; - dtype[2] = '8'; - - } - dtype[3] = '\0'; - chunkXSize = 256; - chunkYSize = 256; - chunkZSize = 256; - order = 'F'; - - char* zArray = ".zarray"; - char* fnFull = (char*)malloc(strlen(folderName)+9); - fnFull[0] = '\0'; - char fileSepS[2]; - fileSepS[0] = '/'; - fileSepS[1] = '\0'; - - strcat(fnFull,folderName); - strcat(fnFull,fileSepS); - strcat(fnFull,zArray); - - // lz4 for the compressor if none is specified - if(!cname) cname = "lz4"; - if(!crop){ - uint8_t nDims = (uint8_t)mxGetNumberOfDimensions(prhs[1]); - if(nDims < 2 || nDims > 3) mexErrMsgIdAndTxt("zarr:inputError","Input data must be 2D or 3D"); - uint64_t* dims = (uint64_t*)mxGetDimensions(prhs[1]); - shapeX = dims[0]; - shapeY = dims[1]; - if(nDims == 3) shapeZ = dims[2]; - else shapeZ = 1; - chunkXSize = (uint64_t)*(mxGetPr(prhs[3])); - chunkYSize = (uint64_t)*((mxGetPr(prhs[3])+1)); - chunkZSize = (uint64_t)*((mxGetPr(prhs[3])+2)); - - FILE* f = fopen(fnFull,"r"); - if(f) fclose(f); - else{ - #ifdef _WIN32 - mkdir(folderName); - #else - mkdir(folderName, 0775); - #endif - chmod(folderName, 0775); - createSubfolders(folderName,shapeX,shapeY,shapeZ,chunkXSize,chunkYSize,chunkZSize,subfolderSizeX,subfolderSizeY,subfolderSizeZ); - } - - - setJSONValues(folderName,&chunkXSize,&chunkYSize,&chunkZSize,dtype,&order,&shapeX,&shapeY,&shapeZ,cname,&clevel,&subfolderSizeX,&subfolderSizeY,&subfolderSizeZ); - setValuesFromJSON(folderName,&chunkXSize,&chunkYSize,&chunkZSize,dtype,&order,&shapeX,&shapeY,&shapeZ,&cname,&clevel,&subfolderSizeX,&subfolderSizeY,&subfolderSizeZ); - //} - - } - else{ - shapeX = endX; - shapeY = endY; - shapeZ = endZ; - - FILE* f = fopen(fnFull,"r"); - if(f){ - fclose(f); - if(!iDims) mexErrMsgIdAndTxt("zarr:inputError","Unable to get input dimensions"); - if(endX-startX != iDims[0] || endY-startY != iDims[1] || endZ-startZ != iDims[2]) mexErrMsgIdAndTxt("zarr:inputError","Bounding box size does not match the size of the input data"); - } - else { - #ifdef _WIN32 - mkdir(folderName); - #else - mkdir(folderName, 0775); - #endif - chmod(folderName, 0775); - createSubfolders(folderName,shapeX,shapeY,shapeZ,chunkXSize,chunkYSize,chunkZSize,subfolderSizeX,subfolderSizeY,subfolderSizeZ); - setJSONValues(folderName,&chunkXSize,&chunkYSize,&chunkZSize,dtype,&order,&shapeX,&shapeY, &shapeZ,cname,&clevel,&subfolderSizeX,&subfolderSizeY,&subfolderSizeZ); - } - - - char dtypeT[4]; - for(int i = 0; i < 4; i++) dtypeT[i] = dtype[i]; - - setValuesFromJSON(folderName,&chunkXSize,&chunkYSize,&chunkZSize,dtype,&order,&shapeX,&shapeY,&shapeZ,&cname,&clevel,&subfolderSizeX,&subfolderSizeY,&subfolderSizeZ); - if(dtypeT[2] != dtype[2]){ - uint64_t size = (endX-startX)*(endY-startY)*(endZ-startZ); - - uint64_t bitsT = 0; - if(dtypeT[1] == 'u' && dtypeT[2] == '1') bitsT = 8; - else if(dtypeT[1] == 'u' && dtypeT[2] == '2') bitsT = 16; - else if(dtypeT[1] == 'f' && dtypeT[2] == '4') bitsT = 32; - else if(dtypeT[1] == 'f' && dtypeT[2] == '8') bitsT = 64; - else mexErrMsgIdAndTxt("tiff:dataTypeError","Cannont convert to passed in data type. Data type not suppported"); - - - if(dtype[1] == 'u' && dtype[2] == '1'){ - zarrC = malloc(size*sizeof(uint8_t)); - if(bitsT == 16){ - uint16_t* zarrT = (uint16_t*)mxGetPr(prhs[1]); - #pragma omp parallel for - for(uint64_t i = 0; i < size; i++){ - ((uint8_t*)zarrC)[i] = (uint8_t)zarrT[i]; - } - } - else if(bitsT == 32){ - float* zarrT = (float*)mxGetPr(prhs[1]); - #pragma omp parallel for - for(uint64_t i = 0; i < size; i++){ - ((uint8_t*)zarrC)[i] = (uint8_t)zarrT[i]; - } - } - else if(bitsT == 64){ - double* zarrT = (double*)mxGetPr(prhs[1]); - #pragma omp parallel for - for(uint64_t i = 0; i < size; i++){ - ((uint8_t*)zarrC)[i] = (uint8_t)zarrT[i]; - } - } - } - else if(dtype[1] == 'u' && dtype[2] == '2'){ - zarrC = malloc(size*sizeof(uint16_t)); - if(bitsT == 8){ - uint8_t* zarrT = (uint8_t*)mxGetPr(prhs[1]); - #pragma omp parallel for - for(uint64_t i = 0; i < size; i++){ - ((uint16_t*)zarrC)[i] = (uint16_t)zarrT[i]; - } - } - else if (bitsT == 32){ - float* zarrT = (float*)mxGetPr(prhs[1]); - #pragma omp parallel for - for(uint64_t i = 0; i < size; i++){ - ((uint16_t*)zarrC)[i] = (uint16_t)zarrT[i]; - } - } - else if (bitsT == 64){ - double* zarrT = (double*)mxGetPr(prhs[1]); - #pragma omp parallel for - for(uint64_t i = 0; i < size; i++){ - ((uint16_t*)zarrC)[i] = (uint16_t)zarrT[i]; - } - } - } - else if(dtype[1] == 'f' && dtype[2] == '4'){ - zarrC = malloc(size*sizeof(float)); - if(bitsT == 8){ - uint8_t* zarrT = (uint8_t*)mxGetPr(prhs[1]); - #pragma omp parallel for - for(uint64_t i = 0; i < size; i++){ - ((float*)zarrC)[i] = (float)zarrT[i]; - } - } - else if(bitsT == 16){ - uint16_t* zarrT = (uint16_t*)mxGetPr(prhs[1]); - #pragma omp parallel for - for(uint64_t i = 0; i < size; i++){ - ((float*)zarrC)[i] = (float)zarrT[i]; - } - } - else if(bitsT == 64){ - double* zarrT = (double*)mxGetPr(prhs[1]); - #pragma omp parallel for - for(uint64_t i = 0; i < size; i++){ - ((float*)zarrC)[i] = (float)zarrT[i]; - } - } - } - else if(dtype[1] == 'f' && dtype[2] == '8'){ - zarrC = malloc(size*sizeof(double)); - if(bitsT == 8){ - uint8_t* zarrT = (uint8_t*)mxGetPr(prhs[1]); - #pragma omp parallel for - for(uint64_t i = 0; i < size; i++){ - ((double*)zarrC)[i] = (double)zarrT[i]; - } - } - else if(bitsT == 16){ - uint16_t* zarrT = (uint16_t*)mxGetPr(prhs[1]); - #pragma omp parallel for - for(uint64_t i = 0; i < size; i++){ - ((double*)zarrC)[i] = (double)zarrT[i]; - } - } - else if(bitsT == 32){ - float* zarrT = (float*)mxGetPr(prhs[1]); - #pragma omp parallel for - for(uint64_t i = 0; i < size; i++){ - ((double*)zarrC)[i] = (double)zarrT[i]; - } - } - } - else{ - mexErrMsgIdAndTxt("zarr:dataTypeError","Cannont convert to passed in data type. Data type not suppported"); - } - } - } - - free(fnFull); - uint64_t origShapeX = shapeX; - uint64_t origShapeY = shapeY; - uint64_t origShapeZ = shapeZ; - - fflush(stdout); - if(endX > shapeX || endY > shapeY || endZ > shapeZ) mexErrMsgIdAndTxt("zarr:inputError","Upper bound is invalid"); - if(!crop){ - endX = shapeX; - endY = shapeY; - endZ = shapeZ; - startX = 0; - startY = 0; - startZ = 0; - } - uint64_t dim[3]; - shapeX = endX-startX; - shapeY = endY-startY; - shapeZ = endZ-startZ; - dim[0] = shapeX; - dim[1] = shapeY; - dim[2] = shapeZ; - - if(dtype[1] == 'u' && dtype[2] == '1'){ - uint64_t bits = 8; - uint8_t* zarr; - if(zarrC) zarr = (uint8_t*)zarrC; - else zarr = (uint8_t*)mxGetPr(prhs[1]); - parallelWriteZarrMex((void*)zarr,folderName,startX,startY,startZ,endX,endY,endZ,chunkXSize,chunkYSize,chunkZSize,shapeX,shapeY,shapeZ, origShapeX, origShapeY,origShapeZ, bits,order,useUuid,crop,cname,clevel,subfolderSizeX,subfolderSizeY,subfolderSizeZ); - } - else if(dtype[1] == 'u' && dtype[2] == '2'){ - uint64_t bits = 16; - uint16_t* zarr; - if(zarrC) zarr = (uint16_t*)zarrC; - else zarr = (uint16_t*)mxGetPr(prhs[1]); - parallelWriteZarrMex((void*)zarr,folderName,startX,startY,startZ,endX,endY,endZ,chunkXSize,chunkYSize,chunkZSize,shapeX,shapeY,shapeZ, origShapeX, origShapeY,origShapeZ, bits,order,useUuid,crop,cname,clevel,subfolderSizeX,subfolderSizeY,subfolderSizeZ); - } - else if(dtype[1] == 'f' && dtype[2] == '4'){ - uint64_t bits = 32; - float* zarr; - if(zarrC) zarr = (float*)zarrC; - else zarr = (float*)mxGetPr(prhs[1]); - parallelWriteZarrMex((void*)zarr,folderName,startX,startY,startZ,endX,endY,endZ,chunkXSize,chunkYSize,chunkZSize,shapeX,shapeY,shapeZ, origShapeX, origShapeY,origShapeZ, bits,order,useUuid,crop,cname,clevel,subfolderSizeX,subfolderSizeY,subfolderSizeZ); - } - else if(dtype[1] == 'f' && dtype[2] == '8'){ - uint64_t bits = 64; - double* zarr; - if(zarrC) zarr = (double*)zarrC; - else zarr = (double*)mxGetPr(prhs[1]); - parallelWriteZarrMex((void*)zarr,folderName,startX,startY,startZ,endX,endY,endZ,chunkXSize,chunkYSize,chunkZSize,shapeX,shapeY,shapeZ, origShapeX, origShapeY,origShapeZ, bits,order,useUuid,crop,cname,clevel,subfolderSizeX,subfolderSizeY,subfolderSizeZ); - } - else{ - free(zarrC); - mexErrMsgIdAndTxt("tiff:dataTypeError","Data type not suppported"); - } - - // zarrC is either a copy for data conversion or NULL - free(zarrC); -} - diff --git a/parallelWriteZarr/parallelWriteZarr.h b/parallelWriteZarr/parallelWriteZarr.h deleted file mode 100755 index df6c02a..0000000 --- a/parallelWriteZarr/parallelWriteZarr.h +++ /dev/null @@ -1,7 +0,0 @@ -#ifndef PARALLELWRITEZARR_H -#define PARALLELWRITEZARR_H -#include - -void parallelWriteZarrMex(void* zarr, char* folderName,uint64_t startX, uint64_t startY, uint64_t startZ, uint64_t endX, uint64_t endY,uint64_t endZ,uint64_t chunkXSize,uint64_t chunkYSize,uint64_t chunkZSize,uint64_t shapeX,uint64_t shapeY,uint64_t shapeZ,uint64_t origShapeX,uint64_t origShapeY,uint64_t origShapeZ, uint64_t bits, char order, uint8_t useUuid, uint8_t crop, char* cname, uint64_t clevel, uint64_t subfolderSizeX, uint64_t subfolderSizeY, uint64_t subfolderSizeZ); - -#endif //PARALLELWRITEZARR_H diff --git a/parallelWriteZarr/windows/blosc.dll b/parallelWriteZarr/windows/blosc.dll deleted file mode 100755 index bec2342..0000000 Binary files a/parallelWriteZarr/windows/blosc.dll and /dev/null differ diff --git a/parallelWriteZarr/windows/cjson.dll b/parallelWriteZarr/windows/cjson.dll deleted file mode 100755 index 4e3efdc..0000000 Binary files a/parallelWriteZarr/windows/cjson.dll and /dev/null differ diff --git a/parallelWriteZarr/windows/cjson_utils.dll b/parallelWriteZarr/windows/cjson_utils.dll deleted file mode 100755 index f395933..0000000 Binary files a/parallelWriteZarr/windows/cjson_utils.dll and /dev/null differ diff --git a/parallelWriteZarr/windows/concrt140.dll b/parallelWriteZarr/windows/concrt140.dll deleted file mode 100755 index 9f2bbff..0000000 Binary files a/parallelWriteZarr/windows/concrt140.dll and /dev/null differ diff --git a/parallelWriteZarr/windows/libblosc2.dll b/parallelWriteZarr/windows/libblosc2.dll deleted file mode 100755 index ad8e48b..0000000 Binary files a/parallelWriteZarr/windows/libblosc2.dll and /dev/null differ diff --git a/parallelWriteZarr/windows/msvcp140.dll b/parallelWriteZarr/windows/msvcp140.dll deleted file mode 100755 index 654233d..0000000 Binary files a/parallelWriteZarr/windows/msvcp140.dll and /dev/null differ diff --git a/parallelWriteZarr/windows/msvcp140_1.dll b/parallelWriteZarr/windows/msvcp140_1.dll deleted file mode 100755 index 1bf95c8..0000000 Binary files a/parallelWriteZarr/windows/msvcp140_1.dll and /dev/null differ diff --git a/parallelWriteZarr/windows/msvcp140_2.dll b/parallelWriteZarr/windows/msvcp140_2.dll deleted file mode 100755 index eab208e..0000000 Binary files a/parallelWriteZarr/windows/msvcp140_2.dll and /dev/null differ diff --git a/parallelWriteZarr/windows/msvcp140_atomic_wait.dll b/parallelWriteZarr/windows/msvcp140_atomic_wait.dll deleted file mode 100755 index 7c4df4f..0000000 Binary files a/parallelWriteZarr/windows/msvcp140_atomic_wait.dll and /dev/null differ diff --git a/parallelWriteZarr/windows/msvcp140_codecvt_ids.dll b/parallelWriteZarr/windows/msvcp140_codecvt_ids.dll deleted file mode 100755 index 40d8eea..0000000 Binary files a/parallelWriteZarr/windows/msvcp140_codecvt_ids.dll and /dev/null differ diff --git a/parallelWriteZarr/windows/parallelWriteZarr.mexw64 b/parallelWriteZarr/windows/parallelWriteZarr.mexw64 deleted file mode 100755 index ba79458..0000000 Binary files a/parallelWriteZarr/windows/parallelWriteZarr.mexw64 and /dev/null differ diff --git a/parallelWriteZarr/windows/vcruntime140.dll b/parallelWriteZarr/windows/vcruntime140.dll deleted file mode 100755 index 931bce0..0000000 Binary files a/parallelWriteZarr/windows/vcruntime140.dll and /dev/null differ diff --git a/parallelWriteZarr/windows/vcruntime140_1.dll b/parallelWriteZarr/windows/vcruntime140_1.dll deleted file mode 100755 index e548262..0000000 Binary files a/parallelWriteZarr/windows/vcruntime140_1.dll and /dev/null differ