forked from scipy/scipy
/
compressed.py
693 lines (539 loc) · 24 KB
/
compressed.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
"""Base class for sparse matrix formats using compressed storage
"""
from __future__ import division, print_function, absolute_import
__all__ = []
from warnings import warn
import numpy as np
from scipy.lib.six.moves import xrange
from .base import spmatrix, isspmatrix, SparseEfficiencyWarning
from .data import _data_matrix, _minmax_mixin
from . import sparsetools
from .sputils import upcast, upcast_char, to_native, isdense, isshape, \
getdtype, isscalarlike, isintlike
class _cs_matrix(_data_matrix, _minmax_mixin):
"""base matrix class for compressed row and column oriented matrices"""
def __init__(self, arg1, shape=None, dtype=None, copy=False):
_data_matrix.__init__(self)
if isspmatrix(arg1):
if arg1.format == self.format and copy:
arg1 = arg1.copy()
else:
arg1 = arg1.asformat(self.format)
self._set_self( arg1 )
elif isinstance(arg1, tuple):
if isshape(arg1):
# It's a tuple of matrix dimensions (M, N)
# create empty matrix
self.shape = arg1 #spmatrix checks for errors here
M, N = self.shape
self.data = np.zeros(0, getdtype(dtype, default=float))
self.indices = np.zeros(0, np.intc)
self.indptr = np.zeros(self._swap((M,N))[0] + 1, dtype=np.intc)
else:
if len(arg1) == 2:
# (data, ij) format
from .coo import coo_matrix
other = self.__class__( coo_matrix(arg1, shape=shape) )
self._set_self( other )
elif len(arg1) == 3:
# (data, indices, indptr) format
(data, indices, indptr) = arg1
self.indices = np.array(indices, copy=copy)
self.indptr = np.array(indptr, copy=copy)
self.data = np.array(data, copy=copy, dtype=getdtype(dtype, data))
else:
raise ValueError("unrecognized %s_matrix constructor usage" %
self.format)
else:
#must be dense
try:
arg1 = np.asarray(arg1)
except:
raise ValueError("unrecognized %s_matrix constructor usage" %
self.format)
from .coo import coo_matrix
self._set_self( self.__class__(coo_matrix(arg1, dtype=dtype)) )
# Read matrix dimensions given, if any
if shape is not None:
self.shape = shape # spmatrix will check for errors
else:
if self.shape is None:
# shape not already set, try to infer dimensions
try:
major_dim = len(self.indptr) - 1
minor_dim = self.indices.max() + 1
except:
raise ValueError('unable to infer matrix dimensions')
else:
self.shape = self._swap((major_dim,minor_dim))
if dtype is not None:
self.data = self.data.astype(dtype)
self.check_format(full_check=False)
def getnnz(self):
return self.indptr[-1]
nnz = property(fget=getnnz)
def _set_self(self, other, copy=False):
"""take the member variables of other and assign them to self"""
if copy:
other = other.copy()
self.data = other.data
self.indices = other.indices
self.indptr = other.indptr
self.shape = other.shape
def check_format(self, full_check=True):
"""check whether the matrix format is valid
Parameters
==========
- full_check : {bool}
- True - rigorous check, O(N) operations : default
- False - basic check, O(1) operations
"""
#use _swap to determine proper bounds
major_name,minor_name = self._swap(('row','column'))
major_dim,minor_dim = self._swap(self.shape)
# index arrays should have integer data types
if self.indptr.dtype.kind != 'i':
warn("indptr array has non-integer dtype (%s)" \
% self.indptr.dtype.name )
if self.indices.dtype.kind != 'i':
warn("indices array has non-integer dtype (%s)" \
% self.indices.dtype.name )
# only support 32-bit ints for now
self.indptr = np.asarray(self.indptr, dtype=np.intc)
self.indices = np.asarray(self.indices, dtype=np.intc)
self.data = to_native(self.data)
# check array shapes
if np.rank(self.data) != 1 or np.rank(self.indices) != 1 or np.rank(self.indptr) != 1:
raise ValueError('data, indices, and indptr should be rank 1')
# check index pointer
if (len(self.indptr) != major_dim + 1 ):
raise ValueError("index pointer size (%d) should be (%d)" %
(len(self.indptr), major_dim + 1))
if (self.indptr[0] != 0):
raise ValueError("index pointer should start with 0")
# check index and data arrays
if (len(self.indices) != len(self.data)):
raise ValueError("indices and data should have the same size")
if (self.indptr[-1] > len(self.indices)):
raise ValueError("Last value of index pointer should be less than "
"the size of index and data arrays")
self.prune()
if full_check:
#check format validity (more expensive)
if self.nnz > 0:
if self.indices.max() >= minor_dim:
raise ValueError("%s index values must be < %d" %
(minor_name,minor_dim))
if self.indices.min() < 0:
raise ValueError("%s index values must be >= 0" %
minor_name)
if np.diff(self.indptr).min() < 0:
raise ValueError("index pointer values must form a "
"non-decreasing sequence")
#if not self.has_sorted_indices():
# warn('Indices were not in sorted order. Sorting indices.')
# self.sort_indices()
# assert(self.has_sorted_indices())
#TODO check for duplicates?
def __add__(self,other):
# First check if argument is a scalar
if isscalarlike(other):
if other == 0:
return self.copy()
else: # Now we would add this scalar to every element.
raise NotImplementedError('adding a nonzero scalar to a '
'sparse matrix is not supported')
elif isspmatrix(other):
if (other.shape != self.shape):
raise ValueError("inconsistent shapes")
return self._binopt(other,'_plus_')
elif isdense(other):
# Convert this matrix to a dense matrix and add them
return self.todense() + other
else:
raise NotImplementedError
def __radd__(self,other):
return self.__add__(other)
def __sub__(self,other):
# First check if argument is a scalar
if isscalarlike(other):
if other == 0:
return self.copy()
else: # Now we would add this scalar to every element.
raise NotImplementedError('adding a nonzero scalar to a '
'sparse matrix is not supported')
elif isspmatrix(other):
if (other.shape != self.shape):
raise ValueError("inconsistent shapes")
return self._binopt(other,'_minus_')
elif isdense(other):
# Convert this matrix to a dense matrix and subtract them
return self.todense() - other
else:
raise NotImplementedError
def __rsub__(self,other): # other - self
#note: this can't be replaced by other + (-self) for unsigned types
if isscalarlike(other):
if other == 0:
return -self.copy()
else: # Now we would add this scalar to every element.
raise NotImplementedError('adding a nonzero scalar to a '
'sparse matrix is not supported')
elif isdense(other):
# Convert this matrix to a dense matrix and subtract them
return other - self.todense()
else:
raise NotImplementedError
def __truediv__(self,other):
if isscalarlike(other):
return self * (1./other)
elif isspmatrix(other):
if other.shape != self.shape:
raise ValueError('inconsistent shapes')
return self._binopt(other,'_eldiv_')
else:
raise NotImplementedError
def multiply(self, other):
"""Point-wise multiplication by another matrix
"""
if other.shape != self.shape:
raise ValueError('inconsistent shapes')
if isdense(other):
return np.multiply(self.todense(),other)
else:
other = self.__class__(other)
return self._binopt(other,'_elmul_')
###########################
# Multiplication handlers #
###########################
def _mul_vector(self, other):
M,N = self.shape
# output array
result = np.zeros(M, dtype=upcast_char(self.dtype.char,
other.dtype.char))
# csr_matvec or csc_matvec
fn = getattr(sparsetools,self.format + '_matvec')
fn(M, N, self.indptr, self.indices, self.data, other, result)
return result
def _mul_multivector(self, other):
M,N = self.shape
n_vecs = other.shape[1] #number of column vectors
result = np.zeros((M,n_vecs), dtype=upcast_char(self.dtype.char,
other.dtype.char))
# csr_matvecs or csc_matvecs
fn = getattr(sparsetools,self.format + '_matvecs')
fn(M, N, n_vecs, self.indptr, self.indices, self.data, other.ravel(), result.ravel())
return result
def _mul_sparse_matrix(self, other):
M, K1 = self.shape
K2, N = other.shape
major_axis = self._swap((M,N))[0]
indptr = np.empty(major_axis + 1, dtype=np.intc)
other = self.__class__(other) #convert to this format
fn = getattr(sparsetools, self.format + '_matmat_pass1')
fn( M, N, self.indptr, self.indices, \
other.indptr, other.indices, \
indptr)
nnz = indptr[-1]
indices = np.empty(nnz, dtype=np.intc)
data = np.empty(nnz, dtype=upcast(self.dtype,other.dtype))
fn = getattr(sparsetools, self.format + '_matmat_pass2')
fn( M, N, self.indptr, self.indices, self.data, \
other.indptr, other.indices, other.data, \
indptr, indices, data)
return self.__class__((data,indices,indptr),shape=(M,N))
def diagonal(self):
"""Returns the main diagonal of the matrix
"""
#TODO support k-th diagonal
fn = getattr(sparsetools, self.format + "_diagonal")
y = np.empty( min(self.shape), dtype=upcast(self.dtype) )
fn(self.shape[0], self.shape[1], self.indptr, self.indices, self.data, y)
return y
def sum(self, axis=None):
"""Sum the matrix over the given axis. If the axis is None, sum
over both rows and columns, returning a scalar.
"""
# The spmatrix base class already does axis=0 and axis=1 efficiently
# so we only do the case axis=None here
if axis is None:
return self.data.sum()
else:
return spmatrix.sum(self,axis)
raise ValueError("axis out of bounds")
#######################
# Getting and Setting #
#######################
def __getitem__(self, key):
if isinstance(key, tuple):
row = key[0]
col = key[1]
#TODO implement CSR[ [1,2,3], X ] with sparse matmat
#TODO make use of sorted indices
if isintlike(row) and isintlike(col):
return self._get_single_element(row,col)
else:
major,minor = self._swap((row,col))
if isintlike(major) and isinstance(minor,slice):
minor_shape = self._swap(self.shape)[1]
start, stop, stride = minor.indices(minor_shape)
out_shape = self._swap( (1, stop-start) )
return self._get_slice( major, start, stop, stride, out_shape)
elif isinstance( row, slice) or isinstance(col, slice):
return self._get_submatrix( row, col )
else:
raise NotImplementedError
elif isintlike(key):
return self[key, :]
else:
raise IndexError("invalid index")
def _get_single_element(self,row,col):
M, N = self.shape
if (row < 0):
row += M
if (col < 0):
col += N
if not (0<=row<M) or not (0<=col<N):
raise IndexError("index out of bounds")
major_index, minor_index = self._swap((row,col))
start = self.indptr[major_index]
end = self.indptr[major_index+1]
indxs = np.where(minor_index == self.indices[start:end])[0]
num_matches = len(indxs)
if num_matches == 0:
# entry does not appear in the matrix
return self.dtype.type(0)
elif num_matches == 1:
return self.data[start:end][indxs[0]]
else:
raise ValueError('nonzero entry (%d,%d) occurs more than once' % (row,col))
def _get_slice(self, i, start, stop, stride, shape):
"""Returns a copy of the elements
[i, start:stop:string] for row-oriented matrices
[start:stop:string, i] for column-oriented matrices
"""
if stride != 1:
raise ValueError("slicing with step != 1 not supported")
if stop <= start:
raise ValueError("slice width must be >= 1")
#TODO make [i,:] faster
#TODO implement [i,x:y:z]
indices = []
for ind in xrange(self.indptr[i], self.indptr[i+1]):
if self.indices[ind] >= start and self.indices[ind] < stop:
indices.append(ind)
index = self.indices[indices] - start
data = self.data[indices]
indptr = np.array([0, len(indices)])
return self.__class__((data, index, indptr), shape=shape, \
dtype=self.dtype)
def _get_submatrix( self, slice0, slice1 ):
"""Return a submatrix of this matrix (new matrix is created)."""
slice0, slice1 = self._swap((slice0,slice1))
shape0, shape1 = self._swap(self.shape)
def _process_slice( sl, num ):
if isinstance( sl, slice ):
i0, i1 = sl.start, sl.stop
if i0 is None:
i0 = 0
elif i0 < 0:
i0 = num + i0
if i1 is None:
i1 = num
elif i1 < 0:
i1 = num + i1
return i0, i1
elif np.isscalar( sl ):
if sl < 0:
sl += num
return sl, sl + 1
else:
return sl[0], sl[1]
def _in_bounds( i0, i1, num ):
if not (0<=i0<num) or not (0<i1<=num) or not (i0<i1):
raise IndexError("index out of bounds: 0<=%d<%d, 0<=%d<%d, %d<%d" %
(i0, num, i1, num, i0, i1))
i0, i1 = _process_slice( slice0, shape0 )
j0, j1 = _process_slice( slice1, shape1 )
_in_bounds( i0, i1, shape0 )
_in_bounds( j0, j1, shape1 )
aux = sparsetools.get_csr_submatrix( shape0, shape1,
self.indptr, self.indices,
self.data,
i0, i1, j0, j1 )
data, indices, indptr = aux[2], aux[1], aux[0]
shape = self._swap( (i1 - i0, j1 - j0) )
return self.__class__( (data,indices,indptr), shape=shape )
def __setitem__(self, key, val):
if isinstance(key, tuple):
row,col = key
if not (isscalarlike(row) and isscalarlike(col)):
raise NotImplementedError("Fancy indexing in assignment not "
"supported for csr matrices.")
M, N = self.shape
if (row < 0):
row += M
if (col < 0):
col += N
if not (0<=row<M) or not (0<=col<N):
raise IndexError("index out of bounds")
major_index, minor_index = self._swap((row,col))
start = self.indptr[major_index]
end = self.indptr[major_index+1]
indxs = np.where(minor_index == self.indices[start:end])[0]
num_matches = len(indxs)
if not np.isscalar(val):
raise ValueError('setting an array element with a sequence')
val = self.dtype.type(val)
if num_matches == 0:
#entry not already present
warn('changing the sparsity structure of a %s_matrix is expensive. ' \
'lil_matrix is more efficient.' % self.format, \
SparseEfficiencyWarning)
if self.has_sorted_indices:
# preserve sorted order
newindx = start + self.indices[start:end].searchsorted(minor_index)
else:
newindx = start
val = np.array([val], dtype=self.data.dtype)
minor_index = np.array([minor_index], dtype=self.indices.dtype)
self.data = np.concatenate((self.data[:newindx], val, self.data[newindx:]))
self.indices = np.concatenate((self.indices[:newindx], minor_index, self.indices[newindx:]))
self.indptr = self.indptr.copy()
self.indptr[major_index+1:] += 1
elif num_matches == 1:
#entry appears exactly once
self.data[start:end][indxs[0]] = val
else:
#entry appears more than once
raise ValueError('nonzero entry (%d,%d) occurs more than once'
% (row,col))
self.check_format(full_check=True)
else:
# We should allow slices here!
raise IndexError("invalid index")
######################
# Conversion methods #
######################
def todia(self):
return self.tocoo(copy=False).todia()
def todok(self):
return self.tocoo(copy=False).todok()
def tocoo(self,copy=True):
"""Return a COOrdinate representation of this matrix
When copy=False the index and data arrays are not copied.
"""
major_dim,minor_dim = self._swap(self.shape)
data = self.data
minor_indices = self.indices
if copy:
data = data.copy()
minor_indices = minor_indices.copy()
major_indices = np.empty(len(minor_indices), dtype=np.intc)
sparsetools.expandptr(major_dim,self.indptr,major_indices)
row,col = self._swap((major_indices,minor_indices))
from .coo import coo_matrix
return coo_matrix((data,(row,col)), self.shape)
def toarray(self, order=None, out=None):
"""See the docstring for `spmatrix.toarray`."""
return self.tocoo(copy=False).toarray(order=order, out=out)
##############################################################
# methods that examine or modify the internal data structure #
##############################################################
def eliminate_zeros(self):
"""Remove zero entries from the matrix
The is an *in place* operation
"""
fn = sparsetools.csr_eliminate_zeros
M,N = self._swap(self.shape)
fn( M, N, self.indptr, self.indices, self.data)
self.prune() #nnz may have changed
def sum_duplicates(self):
"""Eliminate duplicate matrix entries by adding them together
The is an *in place* operation
"""
self.sort_indices()
fn = sparsetools.csr_sum_duplicates
M,N = self._swap(self.shape)
fn( M, N, self.indptr, self.indices, self.data)
self.prune() #nnz may have changed
def __get_sorted(self):
"""Determine whether the matrix has sorted indices
Returns
- True: if the indices of the matrix are in sorted order
- False: otherwise
"""
#first check to see if result was cached
if not hasattr(self,'__has_sorted_indices'):
fn = sparsetools.csr_has_sorted_indices
self.__has_sorted_indices = \
fn( len(self.indptr) - 1, self.indptr, self.indices)
return self.__has_sorted_indices
def __set_sorted(self, val):
self.__has_sorted_indices = bool(val)
has_sorted_indices = property(fget=__get_sorted, fset=__set_sorted)
def sorted_indices(self):
"""Return a copy of this matrix with sorted indices
"""
A = self.copy()
A.sort_indices()
return A
# an alternative that has linear complexity is the following
# although the previous option is typically faster
#return self.toother().toother()
def sort_indices(self):
"""Sort the indices of this matrix *in place*
"""
if not self.has_sorted_indices:
fn = sparsetools.csr_sort_indices
fn( len(self.indptr) - 1, self.indptr, self.indices, self.data)
self.has_sorted_indices = True
def prune(self):
"""Remove empty space after all non-zero elements.
"""
major_dim = self._swap(self.shape)[0]
if len(self.indptr) != major_dim + 1:
raise ValueError('index pointer has invalid length')
if len(self.indices) < self.nnz:
raise ValueError('indices array has fewer than nnz elements')
if len(self.data) < self.nnz:
raise ValueError('data array has fewer than nnz elements')
self.data = self.data[:self.nnz]
self.indices = self.indices[:self.nnz]
###################
# utility methods #
###################
# needed by _data_matrix
def _with_data(self,data,copy=True):
"""Returns a matrix with the same sparsity structure as self,
but with different data. By default the structure arrays
(i.e. .indptr and .indices) are copied.
"""
if copy:
return self.__class__((data,self.indices.copy(),self.indptr.copy()), \
shape=self.shape,dtype=data.dtype)
else:
return self.__class__((data,self.indices,self.indptr), \
shape=self.shape,dtype=data.dtype)
def _binopt(self, other, op):
"""apply the binary operation fn to two sparse matrices"""
other = self.__class__(other)
# e.g. csr_plus_csr, csr_minus_csr, etc.
fn = getattr(sparsetools, self.format + op + self.format)
maxnnz = self.nnz + other.nnz
indptr = np.empty_like(self.indptr)
indices = np.empty(maxnnz, dtype=np.intc)
data = np.empty(maxnnz, dtype=upcast(self.dtype,other.dtype))
fn(self.shape[0], self.shape[1], \
self.indptr, self.indices, self.data,
other.indptr, other.indices, other.data,
indptr, indices, data)
actual_nnz = indptr[-1]
indices = indices[:actual_nnz]
data = data[:actual_nnz]
if actual_nnz < maxnnz // 2:
#too much waste, trim arrays
indices = indices.copy()
data = data.copy()
A = self.__class__((data, indices, indptr), shape=self.shape)
return A