Fixes int overflow when requesting large slices

.circleci/config.yml

@@ -49,7 +49,8 @@ aliases:
        source $WORKDIR/miniconda/etc/profile.d/conda.sh
        conda activate test_cdms
        python run_tests.py -H -v2 --subdir $COVERAGE
-       python run_tests.py -H -v2 --subdir -n 1 $COVERAGE --dask
+
+    no_output_timeout: 60m
 
   - &conda_upload
     name: conda_upload

Src/Cdunifmodule.c

@@ -2476,7 +2476,7 @@ PyCdunifVariable_ReadAsArray(PyCdunifVariableObject *self,
 	npy_intp *dims;
 	PyArrayObject *array;
 	int i, d;
-	int nitems;
+	unsigned long nitems;
 	int error = 0;
 	d = 0;
 	nitems = 1;
@@ -2537,7 +2537,7 @@ PyCdunifVariable_ReadAsArray(PyCdunifVariableObject *self,
 	if (nitems > 0) {
 		if ((self->nd == 0) && (array != NULL)) {
 			long zero = 0;
-			int ret;
+			int ret = 0;
 			Py_BEGIN_ALLOW_THREADS
 			;
 			acquire_Cdunif_lock()
@@ -2565,7 +2565,7 @@ PyCdunifVariable_ReadAsArray(PyCdunifVariableObject *self,
 			count = (long *) malloc(self->nd * sizeof(long));
 			stride = (long *) malloc(self->nd * sizeof(long));
 			if (start != NULL && count != NULL && stride != NULL) {
-				int ret;
+				int ret = 0;
 				for (i = 0; i < self->nd; i++) {
 					start[i] = indices[i].start;
 					stride[i] = indices[i].stride;

setup.py

@@ -48,7 +48,7 @@
        package_dir = {'cdms2': 'Lib'},
        include_dirs = ['Include', 'Include/py3c', numpy.lib.utils.get_include()] + cdat_info.cdunif_include_directories,
        scripts = ['Script/cdscan', 'Script/convertcdms.py',"Script/myproxy_logon"],
-       data_files = [("share/cdms2",["share/test_data_files.txt"])],
+       data_files = [("share/cdms2",["share/test_data_files.txt", "share/test_big_data_files.txt"])],
        ext_modules = [Extension('cdms2.Cdunif',
                                 ['Src/Cdunifmodule.c'],
                                 library_dirs = cdat_info.cdunif_library_directories,

share/test_big_data_files.txt

@@ -0,0 +1 @@
+bb48c94069c4e413bc04ce2c172234f9  so_Omon_CESM2_historical_r1i1p1f1_gn_185001-201412.nc

tests/test_big_array.py

@@ -0,0 +1,47 @@
+import os
+import unittest
+import cdms2
+import numpy as np
+import functools
+
+class TestBigData(unittest.TestCase):
+
+    def setUp(self):
+        if not os.path.exists('large_array.nc'):
+            data = np.random.randint(10, size=(66000,128,256), dtype=np.int8)
+            step = 180.0/128.0
+            lat = cdms2.createAxis(np.arange(-90+(step/2), 90, step), id='lat')
+            print(lat.shape)
+            step = 360.0/256.0
+            lon = cdms2.createAxis(np.arange(0+(step/2), 360, step), id='lon')
+            time = cdms2.createAxis(np.arange(0, 66000), id='time')
+            time.units = 'days since 1990'
+
+            var = cdms2.createVariable(data, axes=[time, lat, lon], id='fake')
+
+            with cdms2.open('large_array.nc', 'w') as f:
+                f.write(var)
+
+    def test_read_large_slice(self):
+        with cdms2.open('large_array.nc') as f:
+            data = f('fake', time=slice(0, 65000))
+
+        nitems = functools.reduce(lambda x, y: x * y, data.shape)
+
+        self.assertTrue(nitems < 2147483647)
+        self.assertFalse(np.all(data == 0))
+
+        del data
+
+        with cdms2.open('large_array.nc') as f:
+            data = f('fake')
+
+        nitems = functools.reduce(lambda x, y: x * y, data.shape)
+
+        self.assertTrue(nitems > 2147483647)
+        self.assertFalse(np.all(data == 0))
+
+
+if __name__ == '__main__':
+    suite = unittest.TestLoader().loadTestsFromTestCase(TestBigData)
+    unittest.TextTestRunner(verbosity=2).run(suite)

tests/test_tvariable.py

@@ -186,9 +186,9 @@ def test_setdimattribute1(self):
         self.assertTrue(numpy.array_equal(v2_dim, compare_arr))
 
     def test_setdimattribute2(self):
-        f = cdms2.open('https://aims3.llnl.gov/thredds/dodsC/cmip5_css01_data/cmip5/output1/CNRM-CERFACS/CNRM-CM5/amip/3hr/atmos/cfSites/r1i1p1/v20120406/prw/prw_cfSites_CNRM-CM5_amip_r1i1p1_197901010300-200901010000.nc')
-        v = f('prw', time=slice(0, 10))
-        t = f('time_bnds')[:10]
+        f = self.getDataFile('tas_GFDL-ESM2G_Amon_historical_r1i1p1_198501-200512-clim.nc')
+        v = f('tas_ac', time=slice(0, 10))
+        t = f('lat_bnds')[:10]
         t_bounds = t.getdimattribute(0, 'bounds')
         v.setdimattribute(dim=0, field='bounds', value=t)
         v_dim_attr = v.getdimattribute(0, 'bounds')