From 6c83ff4a722770a38284a090e778408428ef2494 Mon Sep 17 00:00:00 2001 From: Peter Li Date: Sat, 14 Sep 2013 16:24:08 -0700 Subject: [PATCH 01/21] QCML.dims now delegates to Program.dimensions Have to be a little careful that it's okay to set dims before canonicalize. This appears to work though. --- src/qc_lang.py | 18 +----------------- 1 file changed, 1 insertion(+), 17 deletions(-) diff --git a/src/qc_lang.py b/src/qc_lang.py index e479636..674fca5 100644 --- a/src/qc_lang.py +++ b/src/qc_lang.py @@ -20,16 +20,12 @@ # all cases class QCML(object): - """ Must set .dims externally. - """ - def __init__(self, debug = False): self.debug = debug self.state = PARSE self.problem = None self.__codegen = None - self.__dims = {} # keep track of the codegen language self.language = "" @@ -72,22 +68,12 @@ def canonicalize(self): @property def dims(self): - return self.__dims + return self.problem.dimensions @dims.setter def dims(self, dims): if self.state is PARSE: raise Exception("QCML set_dims: No problem currently parsed.") - - required_dims = self.problem.dimensions - - if not required_dims.issubset(set(dims.keys())): - raise Exception("QCML set_dims: Not all required dims are supplied.") - - if any(type(dims[k]) != int for k in required_dims): - raise Exception("QCML set_dims: Not all supplied dims are integer.") - - self.__dims = dims self.problem.dimensions = dims if self.state is COMPLETE: self.state = CODEGEN @@ -99,8 +85,6 @@ def codegen(self, language="python", *args, **kwargs): raise Exception("QCML codegen: No problem currently parsed.") if self.state is CANONICALIZE: raise Exception("QCML codegen: No problem currently canonicalized.") - if not self.__dims: - raise Exception("QCML codegen: No dimensions currently given. Please set the dimensions of the QCML object.") try: codegen_class = SUPPORTED_LANGUAGES[language] From db320670acb54193df49b124b3b419eb244d1130 Mon Sep 17 00:00:00 2001 From: Peter Li Date: Sat, 14 Sep 2013 23:01:37 -0700 Subject: [PATCH 02/21] Playing around with abstract_dim --- src/properties/abstract_dim.py | 53 ++++++++++++++++++++++++++++++++++ src/properties/shape.py | 5 ++-- 2 files changed, 55 insertions(+), 3 deletions(-) create mode 100644 src/properties/abstract_dim.py diff --git a/src/properties/abstract_dim.py b/src/properties/abstract_dim.py new file mode 100644 index 0000000..1347e31 --- /dev/null +++ b/src/properties/abstract_dim.py @@ -0,0 +1,53 @@ +import collections + +def list_product(l): + return reduce(lambda x,y: x * AbstractDim([y]), l, AbstractDim([1])) + + +class AbstractDim(object): + def __init__(self, *args, **kwargs): + self._c = collections.Counter(kwargs) + for a in args: + if isinstance(a, dict): self._c.update(a) + elif isinstance(a, int): self._c.update({1:a}) + else: self._c.update(a) + + @property + def concrete(self): + return len(self._c) == 1 and 1 in self._c + + def __str__(self): + return ' + '.join(map(self._str_term, sorted(self._c.keys()))) + + def _safe_str(self): + """ Protect with parentheses if needed + """ + if len(self._c) > 1: return "(%s)" % self._c + return str(self._c) + + def _str_term(self, key): + if not key in self._c: return '' + if key == 1: return str(self._c[1]) + if self._c[key] == 1: return key + return "%d*%s" % (self._c[key], key) + + def __mul__(self, other): + """ Currently assumes other is an AbstractDim + """ + if self.concrete: + mul = AbstractDim() + for k,v in other.iteritems(): mul[k] = self._c[1]*v + return mul + if other.concrete: + mul = AbstractDim() + for k,v in self.iteritems(): mul[k] = other._c[1]*v + return mul + ops = sorted([self._safe_str(), other._safe_str()]) + mulkey = "%s * %s" % (ops[0], ops[1]) + return AbstractDim([mulkey]) + + def __add__(self, other): + return AbstractDim(self._c + other._c) + +if __name__ == "__main__": + print AbstractDim('a', 3, a=3) diff --git a/src/properties/shape.py b/src/properties/shape.py index 2e5e6d7..f6577eb 100644 --- a/src/properties/shape.py +++ b/src/properties/shape.py @@ -1,4 +1,5 @@ import itertools +from abstract_dim import AbstractDim from .. helpers import use """ Shape class (and its helpers) @@ -86,9 +87,7 @@ def _check_instantiation(self): self.instantiated = all(type(elem) == int for elem in self.dimensions) def size(self): - if self.instantiated: - return reduce(lambda x,y: x*y, self.dimensions, 1) - raise ValueError("Cannot compute size of abstract dimension") + return AbstractDim(self.dimensions) def __str__(self): if isscalar(self): return "Scalar()" From e3bb5d69cfdb3f4a1a723f4a5b2418082c4de2aa Mon Sep 17 00:00:00 2001 From: Peter Li Date: Sun, 15 Sep 2013 00:58:17 -0700 Subject: [PATCH 03/21] Trying to use AbstractDims but with concrete dims --- src/codegens/C/codegen.py | 2 +- src/codes/encoders/c_encoder.py | 4 ++- src/codes/encoders/matlab_encoder.py | 4 +-- src/codes/encoders/python_encoder.py | 2 +- src/properties/abstract_dim.py | 37 ++++++++++++++++++++++------ src/properties/shape.py | 4 +-- 6 files changed, 38 insertions(+), 15 deletions(-) diff --git a/src/codegens/C/codegen.py b/src/codegens/C/codegen.py index baec661..40c236b 100644 --- a/src/codegens/C/codegen.py +++ b/src/codegens/C/codegen.py @@ -142,7 +142,7 @@ def c_cone_sizes(self): yield "q_ptr = data->q;" for num, sz in self.cone_list: if num == 1: yield "*q_ptr++ = %s;" % sz - else: yield "for(i = 0; i < %d; ++i) *q_ptr++ = %s;" % (num, sz) + else: yield "for(i = 0; i < %s; ++i) *q_ptr++ = %s;" % (num, sz) # function to get parameters def c_params(self, program_node): diff --git a/src/codes/encoders/c_encoder.py b/src/codes/encoders/c_encoder.py index 6937c3f..b5963fa 100644 --- a/src/codes/encoders/c_encoder.py +++ b/src/codes/encoders/c_encoder.py @@ -1,5 +1,6 @@ from . encoder import create_encoder from ... import codes +from ... properties.abstract_dim import AbstractDim """ In this file, you'll often see strings with "%%(ptr)s"; this is to delay the evaluation of the ptr string until after the object has been turned @@ -84,7 +85,8 @@ def nnz(x): codes.Assign: assign, codes.NNZ: nnz, str: lambda x: x, - int: lambda x: str(x) + int: lambda x: str(x), + AbstractDim: lambda x: str(x) } toC = create_encoder(lookup) diff --git a/src/codes/encoders/matlab_encoder.py b/src/codes/encoders/matlab_encoder.py index 8755c52..8eddf30 100644 --- a/src/codes/encoders/matlab_encoder.py +++ b/src/codes/encoders/matlab_encoder.py @@ -12,7 +12,7 @@ def constant(x): return str(x.value) def eye(x): - return "%s * speye(%d)" % (toMatlab(x.coeff), x.n) + return "%s * speye(%s)" % (toMatlab(x.coeff), x.n) def ones(x): if x.transpose: return "%s * ones(1,%d)" % (toMatlab(x.coeff), x.n) @@ -69,7 +69,7 @@ def _range(x): else: return "(%d:%d:%d)'" % (x.start, x.stride, x.end-1) def repeat(x): - return "%s*ones(%d,1)" % (toMatlab(x.obj), x.n) + return "%s*ones(%s,1)" % (toMatlab(x.obj), x.n) def assign(x): return "%s = %s" % (toMatlab(x.lhs), toMatlab(x.rhs)) diff --git a/src/codes/encoders/python_encoder.py b/src/codes/encoders/python_encoder.py index 61dd923..d344850 100644 --- a/src/codes/encoders/python_encoder.py +++ b/src/codes/encoders/python_encoder.py @@ -46,7 +46,7 @@ def _range(x): return "xrange(%d, %d, %d)" % (x.start, x.end, x.stride) def repeat(x): - return "itertools.repeat(%s, %d)" % (toPython(x.obj), x.n) + return "itertools.repeat(%s, %s)" % (toPython(x.obj), x.n) def assign(x): return "%s = sp.coo_matrix(%s)" % (toPython(x.lhs), toPython(x.rhs)) diff --git a/src/properties/abstract_dim.py b/src/properties/abstract_dim.py index 1347e31..00c34f9 100644 --- a/src/properties/abstract_dim.py +++ b/src/properties/abstract_dim.py @@ -1,8 +1,7 @@ import collections def list_product(l): - return reduce(lambda x,y: x * AbstractDim([y]), l, AbstractDim([1])) - + return reduce(lambda x,y: x * AbstractDim(y), l, AbstractDim(1)) class AbstractDim(object): def __init__(self, *args, **kwargs): @@ -10,6 +9,7 @@ def __init__(self, *args, **kwargs): for a in args: if isinstance(a, dict): self._c.update(a) elif isinstance(a, int): self._c.update({1:a}) + elif isinstance(a, str): self._c.update([a]) else: self._c.update(a) @property @@ -22,8 +22,8 @@ def __str__(self): def _safe_str(self): """ Protect with parentheses if needed """ - if len(self._c) > 1: return "(%s)" % self._c - return str(self._c) + if len(self._c) > 1: return "(%s)" % self + return str(self) def _str_term(self, key): if not key in self._c: return '' @@ -36,18 +36,39 @@ def __mul__(self, other): """ if self.concrete: mul = AbstractDim() - for k,v in other.iteritems(): mul[k] = self._c[1]*v + for k,v in other._c.iteritems(): mul._c[k] = self._c[1]*v return mul if other.concrete: mul = AbstractDim() - for k,v in self.iteritems(): mul[k] = other._c[1]*v + for k,v in self._c.iteritems(): mul._c[k] = other._c[1]*v return mul ops = sorted([self._safe_str(), other._safe_str()]) mulkey = "%s * %s" % (ops[0], ops[1]) - return AbstractDim([mulkey]) + return AbstractDim(mulkey) def __add__(self, other): return AbstractDim(self._c + other._c) + def __sub__(self, other): + return AbstractDim(self._c - other._c) + + def __radd__(self, other): + """ Currently assumes other is int + """ + if self.concrete: + return other + self._c[1] + return AbstractDim(other) + self + + def __rmul__(self, other): + """ Currently assumes other is int + """ + if self.concrete: + return other * self._c[1] + return AbstractDim(other) * self + if __name__ == "__main__": - print AbstractDim('a', 3, a=3) + lp = list_product(['n', 3]) + m = AbstractDim('m', [1, 1, 1, 1]) + print lp + print m + print lp * m diff --git a/src/properties/shape.py b/src/properties/shape.py index f6577eb..80879df 100644 --- a/src/properties/shape.py +++ b/src/properties/shape.py @@ -1,5 +1,5 @@ import itertools -from abstract_dim import AbstractDim +import abstract_dim from .. helpers import use """ Shape class (and its helpers) @@ -87,7 +87,7 @@ def _check_instantiation(self): self.instantiated = all(type(elem) == int for elem in self.dimensions) def size(self): - return AbstractDim(self.dimensions) + return abstract_dim.list_product(self.dimensions) def __str__(self): if isscalar(self): return "Scalar()" From 0ee83884cd7971f0aaf18db017220b75947af7bd Mon Sep 17 00:00:00 2001 From: Peter Li Date: Sun, 15 Sep 2013 01:26:02 -0700 Subject: [PATCH 04/21] Added __eq__ to AbstractDim to support expr simplifications So that AbstractDim(1) == 1 -> True and AbstractDim(-1) == -1 -> True. Useful for simplifying expressions. --- src/codes/encoders/c_encoder.py | 2 +- src/properties/abstract_dim.py | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/src/codes/encoders/c_encoder.py b/src/codes/encoders/c_encoder.py index b5963fa..a42fe31 100644 --- a/src/codes/encoders/c_encoder.py +++ b/src/codes/encoders/c_encoder.py @@ -58,7 +58,7 @@ def _range(x): return "for(i = %d; i < %d; i+=%d) *%%(ptr)s++ = i;" % (x.start, x.end, x.stride) def repeat(x): - return "for(i = 0; i < %d; ++i) *%%(ptr)s++ = %s;" % (x.n, toC(x.obj)) + return "for(i = 0; i < %s; ++i) *%%(ptr)s++ = %s;" % (x.n, toC(x.obj)) def assign(x): raise Exception("Assignment not implemented.... %s = %s" % (x.lhs, x.rhs)) diff --git a/src/properties/abstract_dim.py b/src/properties/abstract_dim.py index 00c34f9..2f53a2f 100644 --- a/src/properties/abstract_dim.py +++ b/src/properties/abstract_dim.py @@ -31,6 +31,14 @@ def _str_term(self, key): if self._c[key] == 1: return key return "%d*%s" % (self._c[key], key) + def __eq__(self, other): + """ Coefficient operations like codegen_mul check whether expressions == 1 or == -1 to allow simplifications. So we want to be able to have + AbstractDim(1) == 1 -> True + """ + if self.concrete and isinstance(other, int): + return self._c[1] == other + return self._c == other._c + def __mul__(self, other): """ Currently assumes other is an AbstractDim """ From fbbb10b9768391df7c076d60610bdf8d6d2c8184 Mon Sep 17 00:00:00 2001 From: Peter Li Date: Sun, 15 Sep 2013 01:39:39 -0700 Subject: [PATCH 05/21] Work in progress: abstract dim generates Matlab code! So actually spits out Matlab code with abstract dimensions! Probably there are bugs though, especially around proper parenthesizing; should fold AbstractDim._safe_str into __str__ so that parentheses get added under most printing conditions and not just when multiplying/dividing. Shouldn't take too much to get this working for Python and C codegens too. Things are generally brittle; I've only implemented the AbstractDim operators that seemed to be necessary for the CBP example, so there are likely to be more needed to cover general case. --- examples/cbp.py | 2 +- src/codegens/matlab/codegen.py | 8 +++---- src/codes/encoders/matlab_encoder.py | 8 +++---- src/properties/abstract_dim.py | 32 ++++++++++++++++++++++++++-- 4 files changed, 39 insertions(+), 11 deletions(-) diff --git a/examples/cbp.py b/examples/cbp.py index 0f569fa..006aed6 100755 --- a/examples/cbp.py +++ b/examples/cbp.py @@ -56,7 +56,7 @@ p.canonicalize() raw_input("press ENTER to generate code....") - p.dims = {'n': n, 'm': m} +# p.dims = {'n': n, 'm': m} p.codegen(args.codegen) raw_input("press ENTER for raw code....") diff --git a/src/codegens/matlab/codegen.py b/src/codegens/matlab/codegen.py index 23f3c2e..8682666 100644 --- a/src/codegens/matlab/codegen.py +++ b/src/codegens/matlab/codegen.py @@ -29,7 +29,7 @@ def functions_setup(self, program_node): self.prob2socp.document(self.printshapes(program_node)) self.prob2socp.newline() - self.prob2socp.add_lines("p = %d; m = %d; n = %d;" % v for v in self.pmn) + self.prob2socp.add_lines("p = %s; m = %s; n = %s;" % v for v in self.pmn) self.prob2socp.add_lines("c = zeros(n,1);") self.prob2socp.add_lines("h = zeros(m,1);") self.prob2socp.add_lines("b = zeros(p,1);") @@ -37,7 +37,7 @@ def functions_setup(self, program_node): self.prob2socp.add_lines("Ai = []; Aj = []; Av = [];") self.prob2socp.newline() - self.prob2socp.add_lines("dims.l = %d;" % l for l in self.dimsl) + self.prob2socp.add_lines("dims.l = %s;" % l for l in self.dimsl) self.prob2socp.add_lines("dims.q = [%s];" % q for q in self.dimsq()) self.prob2socp.add_lines("dims.s = [];") @@ -61,9 +61,9 @@ def stuff_vec(self, vec, start, end, expr, stride): 0 indexed. Hopefully this can be cleaned up! """ if stride == 1: - yield "%s(%d:%d) = %s;" % (vec, start+1, end, toMatlab(expr)) + yield "%s(%s:%s) = %s;" % (vec, start+1, end, toMatlab(expr)) else: - yield "%s(%d:%d:%d) = %s;" % (vec, start+1, stride, end, toMatlab(expr)) + yield "%s(%s:%s:%s) = %s;" % (vec, start+1, stride, end, toMatlab(expr)) def stuff_c(self, start, end, expr, stride = 1): return self.stuff_vec("c", start, end, expr, stride) diff --git a/src/codes/encoders/matlab_encoder.py b/src/codes/encoders/matlab_encoder.py index 8eddf30..f9f5c1f 100644 --- a/src/codes/encoders/matlab_encoder.py +++ b/src/codes/encoders/matlab_encoder.py @@ -46,7 +46,7 @@ def loop_rows(x): if hasattr(x, 'stride') and x.stride != 1: ret = "%d*%s" % (x.stride, ret) if hasattr(x, 'offset') and x.offset != 0: - ret = "%d + %s" % (x.offset, ret) + ret = "%s + %s" % (x.offset, ret) return ret @@ -57,7 +57,7 @@ def loop_cols(x): if hasattr(x, 'stride') and x.stride != 1: ret = "%d*%s" % (x.stride, ret) if hasattr(x, 'offset') and x.offset != 0: - ret = "%d + %s" % (x.offset, ret) + ret = "%s + %s" % (x.offset, ret) return ret @@ -65,8 +65,8 @@ def loop_over(x): return "nonzeros(%s)" % (x.op % toMatlab(x.matrix)) def _range(x): - if x.stride == 1: return "(%d:%d)'" % (x.start, x.end-1) - else: return "(%d:%d:%d)'" % (x.start, x.stride, x.end-1) + if x.stride == 1: return "(%s:%s)'" % (x.start, x.end-1) + else: return "(%s:%s:%s)'" % (x.start, x.stride, x.end-1) def repeat(x): return "%s*ones(%s,1)" % (toMatlab(x.obj), x.n) diff --git a/src/properties/abstract_dim.py b/src/properties/abstract_dim.py index 2f53a2f..9ed573b 100644 --- a/src/properties/abstract_dim.py +++ b/src/properties/abstract_dim.py @@ -35,8 +35,10 @@ def __eq__(self, other): """ Coefficient operations like codegen_mul check whether expressions == 1 or == -1 to allow simplifications. So we want to be able to have AbstractDim(1) == 1 -> True """ - if self.concrete and isinstance(other, int): - return self._c[1] == other + if isinstance(other, int): + if self.concrete: + return self._c[1] == other + return False return self._c == other._c def __mul__(self, other): @@ -54,10 +56,36 @@ def __mul__(self, other): mulkey = "%s * %s" % (ops[0], ops[1]) return AbstractDim(mulkey) + def __div__(self, other): + if isinstance(other, int): + if self.concrete: + return self._c[1] / other + return self / AbstractDim(other) + + if self.concrete: + mul = AbstractDim() + for k,v in other._c.iteritems(): mul._c[k] = self._c[1]/v + return mul + if other.concrete: + mul = AbstractDim() + for k,v in self._c.iteritems(): mul._c[k] = other._c[1]/v + return mul + mulkey = "%s / %s" % (self._safe_str(), other._safe_str()) + return AbstractDim(mulkey) + + def __add__(self, other): + if isinstance(other, int): + if self.concrete: + return self._c[1] + other + return self + AbstractDim(other) return AbstractDim(self._c + other._c) def __sub__(self, other): + if isinstance(other, int): + if self.concrete: + return self._c[1] - other + return self - AbstractDim(other) return AbstractDim(self._c - other._c) def __radd__(self, other): From 25ea143ef4db9b9afbb9fc9b42dcad8530a37c4b Mon Sep 17 00:00:00 2001 From: Peter Li Date: Sun, 15 Sep 2013 11:10:08 -0700 Subject: [PATCH 06/21] Adapt CBP example for flexdims --- examples/cbp.py | 36 +++++++++++------------ examples/cbp2.py | 74 ++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 92 insertions(+), 18 deletions(-) create mode 100755 examples/cbp2.py diff --git a/examples/cbp.py b/examples/cbp.py index 006aed6..6ecfda3 100755 --- a/examples/cbp.py +++ b/examples/cbp.py @@ -10,11 +10,11 @@ if __name__ == "__main__": parser = ArgumentParser(description='Continuous Basis Pursuit QCML example') - parser.add_argument('m', type=int, help='Length of waveform (samples)') - parser.add_argument('n', type=int, help='Number of templates in dictionary') + parser.add_argument('-q', type=int, help='Length of waveform (samples)') + parser.add_argument('-r', type=int, help='Number of templates in dictionary') parser.add_argument('-c', '--codegen', help='Codegen type to use (python, matlab, or C; default python)', default='python') args = parser.parse_args() - n, m = (args.n, args.m) + q, r = (args.q, args.r) print "Running CBP example...." @@ -22,18 +22,18 @@ p = QCML(debug=True) p.parse(""" - dimensions m n - variable c(n) - variable u(n) - variable v(n) + dimensions q r + variable c(r) + variable u(r) + variable v(r) parameter noise positive - parameter lambda(n) - parameter data(m) - parameter dictc(m,n) - parameter dictu(m,n) - parameter dictv(m,n) - parameter radii(n,n) # diagonal matrix - parameter rctheta(n,n) # diagonal matrix + parameter lambda(r) + parameter data(q) + parameter dictc(q,r) + parameter dictu(q,r) + parameter dictv(q,r) + parameter radii(r,r) # diagonal matrix + parameter rctheta(r,r) # diagonal matrix minimize noise*norm(data - (dictc*c + dictu*u + dictv*v)) + lambda'*c subject to # || (u[i], v[i]) || <= radii_i * c_i @@ -56,12 +56,12 @@ p.canonicalize() raw_input("press ENTER to generate code....") -# p.dims = {'n': n, 'm': m} + if q and r: p.dims = {'q': q, 'r': r} p.codegen(args.codegen) - raw_input("press ENTER for raw code....") - print p.prob2socp.source - print p.socp2prob.source + #raw_input("press ENTER for raw code....") + #print p.prob2socp.source + #print p.socp2prob.source #socp_data = p.prob2socp(params=locals()) #import ecos diff --git a/examples/cbp2.py b/examples/cbp2.py new file mode 100755 index 0000000..a05f733 --- /dev/null +++ b/examples/cbp2.py @@ -0,0 +1,74 @@ +#!/usr/bin/env python +""" Continuous basis pursuit example. + + TODO: (PHLI) fill in reference to Eero's paper + See .... +""" + +from argparse import ArgumentParser +from qcml import QCML + +if __name__ == "__main__": + parser = ArgumentParser(description='Continuous Basis Pursuit QCML example') + parser.add_argument('m', type=int, help='Length of waveform (samples)') + parser.add_argument('n', type=int, help='Number of templates in dictionary') + parser.add_argument('-c', '--codegen', help='Codegen type to use (python, matlab, or C; default python)', default='python') + args = parser.parse_args() + n, m = (args.n, args.m) + + print "Running CBP example...." + + # TODO: takeaways from this example: "diag" constructor? + + p = QCML(debug=True) + p.parse(""" + dimensions m n + variable c(n) + variable u(n) + variable v(n) + parameter noise positive + parameter lambda(n) + parameter data(m) + parameter dictc(m,n) + parameter dictu(m,n) + parameter dictv(m,n) + parameter radii(n,n) # diagonal matrix + parameter rctheta(n,n) # diagonal matrix + minimize noise*norm(data - (dictc*c + dictu*u + dictv*v)) + lambda'*c + subject to + # || (u[i], v[i]) || <= radii_i * c_i + # norm([u_i v_i]) <= radii_i*c_i + # norm(x,y) applies norm across rows of the matrix [x y] + norm(u,v) <= radii*c + + # rctheta is going to be a diagonal matrix + # rctheta[i]*c[i] <= u[i] implemented with rctheta a diag matrix + rctheta*c <= u + + c <= 1.5 + """) + + # More natural formulation would be: + # minimize 1/(sqrt(2)*noisesigma) * (data - (dictc*c + dictu(u + dictv*v)) + lambda'*c) + # subject to + # sqrt(u.^2 + v.^2) <= radii.*c + # radii.*cos(theta) <= u + + raw_input("press ENTER to canonicalize....") + p.canonicalize() + + raw_input("press ENTER to generate code....") + p.dims = {'n': n, 'm': m} + p.codegen(args.codegen) + + raw_input("press ENTER for raw code....") + print p.prob2socp.source + print p.socp2prob.source + + #socp_data = p.prob2socp(params=locals()) + #import ecos + #sol = ecos.ecos(**socp_data) + #my_vars = p.socp2prob(sol['x']) + #pr.disable() + #ps = pstats.Stats(pr) + #ps.sort_stats('cumulative').print_stats(.5) From 686ab0ee65b75b0dfdccd38ad80f9ef76424aac4 Mon Sep 17 00:00:00 2001 From: Peter Li Date: Sun, 15 Sep 2013 11:10:23 -0700 Subject: [PATCH 07/21] Bugfixes for AbstractDim collections.Counter.subtract is implemented in a pretty broken way, but easy to work around. Also I had a bug in the __div__ code. --- src/properties/abstract_dim.py | 53 +++++++++++++++++----------------- 1 file changed, 27 insertions(+), 26 deletions(-) diff --git a/src/properties/abstract_dim.py b/src/properties/abstract_dim.py index 9ed573b..c38f066 100644 --- a/src/properties/abstract_dim.py +++ b/src/properties/abstract_dim.py @@ -17,13 +17,11 @@ def concrete(self): return len(self._c) == 1 and 1 in self._c def __str__(self): - return ' + '.join(map(self._str_term, sorted(self._c.keys()))) - - def _safe_str(self): - """ Protect with parentheses if needed + """ Protect with parentheses if more than one term """ - if len(self._c) > 1: return "(%s)" % self - return str(self) + ret = ' + '.join(map(self._str_term, sorted(self._c.keys()))) + if len(self._c) < 2: return ret + return "(%s)" % ret def _str_term(self, key): if not key in self._c: return '' @@ -35,11 +33,11 @@ def __eq__(self, other): """ Coefficient operations like codegen_mul check whether expressions == 1 or == -1 to allow simplifications. So we want to be able to have AbstractDim(1) == 1 -> True """ - if isinstance(other, int): - if self.concrete: - return self._c[1] == other - return False - return self._c == other._c + if isinstance(other, AbstractDim): + return self._c == other._c + if isinstance(other, int) and self.concrete: + return self._c[1] == other + return False def __mul__(self, other): """ Currently assumes other is an AbstractDim @@ -52,26 +50,28 @@ def __mul__(self, other): mul = AbstractDim() for k,v in self._c.iteritems(): mul._c[k] = other._c[1]*v return mul - ops = sorted([self._safe_str(), other._safe_str()]) + ops = sorted([str(self), str(other)]) mulkey = "%s * %s" % (ops[0], ops[1]) return AbstractDim(mulkey) def __div__(self, other): + print "self: %s" % self + print "other: %s" % other if isinstance(other, int): if self.concrete: return self._c[1] / other return self / AbstractDim(other) if self.concrete: - mul = AbstractDim() - for k,v in other._c.iteritems(): mul._c[k] = self._c[1]/v - return mul + div = AbstractDim() + for k,v in other._c.iteritems(): div._c[k] = self._c[1]/v + return div if other.concrete: - mul = AbstractDim() - for k,v in self._c.iteritems(): mul._c[k] = other._c[1]/v - return mul - mulkey = "%s / %s" % (self._safe_str(), other._safe_str()) - return AbstractDim(mulkey) + div = AbstractDim() + for k,v in self._c.iteritems(): div._c[k] = v/other._c[1] + return div + divkey = "%s / %s" % (self, other) + return AbstractDim(divkey) def __add__(self, other): @@ -86,7 +86,9 @@ def __sub__(self, other): if self.concrete: return self._c[1] - other return self - AbstractDim(other) - return AbstractDim(self._c - other._c) + sub = self._c.copy() + sub.subtract(other._c) + return AbstractDim(sub) def __radd__(self, other): """ Currently assumes other is int @@ -103,8 +105,7 @@ def __rmul__(self, other): return AbstractDim(other) * self if __name__ == "__main__": - lp = list_product(['n', 3]) - m = AbstractDim('m', [1, 1, 1, 1]) - print lp - print m - print lp * m + r = AbstractDim('r') + r1 = r - 1 + print r + print r1 From 6afb367b9abdc4b4feb07279a36d7f4babeb0e0e Mon Sep 17 00:00:00 2001 From: Peter Li Date: Sun, 15 Sep 2013 11:34:39 -0700 Subject: [PATCH 08/21] Raw code output back in --- examples/cbp.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/examples/cbp.py b/examples/cbp.py index 6ecfda3..44152d0 100755 --- a/examples/cbp.py +++ b/examples/cbp.py @@ -59,9 +59,9 @@ if q and r: p.dims = {'q': q, 'r': r} p.codegen(args.codegen) - #raw_input("press ENTER for raw code....") - #print p.prob2socp.source - #print p.socp2prob.source + raw_input("press ENTER for raw code....") + print p.prob2socp.source + print p.socp2prob.source #socp_data = p.prob2socp(params=locals()) #import ecos From 3b7ac3e7002317f016743df37a1fffd99849c176 Mon Sep 17 00:00:00 2001 From: Peter Li Date: Sun, 15 Sep 2013 11:34:58 -0700 Subject: [PATCH 09/21] AbstractDim printing cleanup Don't print terms with 0 coefficients, for an empty AD or one with all 0 coefficients, print 0. --- src/codegens/python/codegen.py | 4 ++-- src/codes/encoders/python_encoder.py | 4 ++-- src/properties/abstract_dim.py | 19 ++++++++++--------- 3 files changed, 14 insertions(+), 13 deletions(-) diff --git a/src/codegens/python/codegen.py b/src/codegens/python/codegen.py index 04eec81..0f6f3ae 100644 --- a/src/codegens/python/codegen.py +++ b/src/codegens/python/codegen.py @@ -25,7 +25,7 @@ def socp2prob(self): # function to get problem dimensions def python_dimensions(self): - yield "p, m, n = %d, %d, %d" % (self.num_lineqs, self.num_conic + self.num_lps, self.num_vars) + yield "p, m, n = %s, %s, %s" % (self.num_lineqs, self.num_conic + self.num_lps, self.num_vars) # function to get cone dimensions def python_cone_sizes(self): @@ -38,7 +38,7 @@ def cone_tuple_to_str(x): cone_list_str = map(cone_tuple_to_str, self.cone_list) cone_list_str = '+'.join(cone_list_str) - yield "dims = {'l': %d, 'q': %s, 's': []}" % (self.num_lps, cone_list_str) + yield "dims = {'l': %s, 'q': %s, 's': []}" % (self.num_lps, cone_list_str) def functions_setup(self, program_node): # add some documentation diff --git a/src/codes/encoders/python_encoder.py b/src/codes/encoders/python_encoder.py index d344850..5f5781e 100644 --- a/src/codes/encoders/python_encoder.py +++ b/src/codes/encoders/python_encoder.py @@ -38,12 +38,12 @@ def to_str(x): if hasattr(x, 'offset') and hasattr(x, 'stride'): if x.offset == 0 and x.stride == 1: return "(idx for idx in %s.%s)" % (matrix, ijv) - return "(%d + %d*idx for idx in %s.%s)" % (x.offset, x.stride, matrix, ijv) + return "(%s + %s*idx for idx in %s.%s)" % (x.offset, x.stride, matrix, ijv) return "(%s for v in %s.%s)" % (x.op % "v", matrix, ijv) return to_str def _range(x): - return "xrange(%d, %d, %d)" % (x.start, x.end, x.stride) + return "xrange(%s, %s, %s)" % (x.start, x.end, x.stride) def repeat(x): return "itertools.repeat(%s, %s)" % (toPython(x.obj), x.n) diff --git a/src/properties/abstract_dim.py b/src/properties/abstract_dim.py index c38f066..41d595c 100644 --- a/src/properties/abstract_dim.py +++ b/src/properties/abstract_dim.py @@ -15,12 +15,18 @@ def __init__(self, *args, **kwargs): @property def concrete(self): return len(self._c) == 1 and 1 in self._c + + def keys(self): + return self._c.keys() def __str__(self): - """ Protect with parentheses if more than one term + """ Protect with parentheses if more than one term. Don't bother to + print terms with coefficient == 0 """ - ret = ' + '.join(map(self._str_term, sorted(self._c.keys()))) - if len(self._c) < 2: return ret + printkeys = filter(lambda x: self._c[x] != 0, self.keys()) + if not printkeys: return "0" + ret = ' + '.join(map(self._str_term, sorted(printkeys))) + if len(printkeys) < 2: return ret return "(%s)" % ret def _str_term(self, key): @@ -55,8 +61,6 @@ def __mul__(self, other): return AbstractDim(mulkey) def __div__(self, other): - print "self: %s" % self - print "other: %s" % other if isinstance(other, int): if self.concrete: return self._c[1] / other @@ -105,7 +109,4 @@ def __rmul__(self, other): return AbstractDim(other) * self if __name__ == "__main__": - r = AbstractDim('r') - r1 = r - 1 - print r - print r1 + print AbstractDim() From 130484f158ab7ee8f3a6ffe01edf1b8947ee9d6f Mon Sep 17 00:00:00 2001 From: Peter Li Date: Sun, 15 Sep 2013 11:59:56 -0700 Subject: [PATCH 10/21] Docu --- src/properties/abstract_dim.py | 53 ++++++++++++++++++++++++++++++---- 1 file changed, 47 insertions(+), 6 deletions(-) diff --git a/src/properties/abstract_dim.py b/src/properties/abstract_dim.py index 41d595c..891213d 100644 --- a/src/properties/abstract_dim.py +++ b/src/properties/abstract_dim.py @@ -1,9 +1,32 @@ import collections def list_product(l): + """ Take a list and return an AD by multiplying list elements together + """ return reduce(lambda x,y: x * AbstractDim(y), l, AbstractDim(1)) class AbstractDim(object): + """ Implemented by holding a collections.Counter to keep track of terms. + + Counter is held by composition rather than inheriting from there; makes + __init__ somewhat simpler and is cleaner. + + The key 1 is used for constant terms. All other keys should be strs + representing abstract terms. + + Designed to have reasonable arithmetic operations with plain ints. + Didn't bother to worry about Python2 long type. Doesn't currently deal + with floats. + + In general plain ints are promoted to ADs in operations, but if the AD + is concrete then it can be converted back into an int. Currently this + is done in most cases, although perhaps would be cleaner not to be so + generous. + + I didn't implement all possible arthmetic or coercions, only the ones + that were needed to get codegen to run on example problems. So there + may be things missing for general case. + """ def __init__(self, *args, **kwargs): self._c = collections.Counter(kwargs) for a in args: @@ -14,22 +37,34 @@ def __init__(self, *args, **kwargs): @property def concrete(self): - return len(self._c) == 1 and 1 in self._c - + """ AD is concrete means it can be converted into an int, i.e. it has + no abstract part with nonzero coefficients. + """ + nzkeys = self.nzkeys() + if len(nzkeys) == 0: return True + return len(nzkeys) == 1 and self._c[1] != 0 + def keys(self): return self._c.keys() + def nzkeys(self): + """ List of keys with nonzero values (i.e. coefficients) + """ + return filter(lambda x: self._c[x] != 0, self.keys()) + def __str__(self): """ Protect with parentheses if more than one term. Don't bother to print terms with coefficient == 0 """ - printkeys = filter(lambda x: self._c[x] != 0, self.keys()) + printkeys = self.nzkeys() if not printkeys: return "0" ret = ' + '.join(map(self._str_term, sorted(printkeys))) if len(printkeys) < 2: return ret return "(%s)" % ret def _str_term(self, key): + """ Format single term nicely + """ if not key in self._c: return '' if key == 1: return str(self._c[1]) if self._c[key] == 1: return key @@ -46,7 +81,7 @@ def __eq__(self, other): return False def __mul__(self, other): - """ Currently assumes other is an AbstractDim + """ Assumes other is an AD """ if self.concrete: mul = AbstractDim() @@ -61,6 +96,8 @@ def __mul__(self, other): return AbstractDim(mulkey) def __div__(self, other): + """ Assumes other is int or AD + """ if isinstance(other, int): if self.concrete: return self._c[1] / other @@ -79,6 +116,8 @@ def __div__(self, other): def __add__(self, other): + """ Assumes other is int or AD + """ if isinstance(other, int): if self.concrete: return self._c[1] + other @@ -86,6 +125,8 @@ def __add__(self, other): return AbstractDim(self._c + other._c) def __sub__(self, other): + """ Assumes other is int or AD + """ if isinstance(other, int): if self.concrete: return self._c[1] - other @@ -95,14 +136,14 @@ def __sub__(self, other): return AbstractDim(sub) def __radd__(self, other): - """ Currently assumes other is int + """ Assumes other is int """ if self.concrete: return other + self._c[1] return AbstractDim(other) + self def __rmul__(self, other): - """ Currently assumes other is int + """ Assumes other is int """ if self.concrete: return other * self._c[1] From 47e028deffb4ba3a20c313e48f44907ae7a1c06f Mon Sep 17 00:00:00 2001 From: Peter Li Date: Sun, 15 Sep 2013 15:09:06 -0700 Subject: [PATCH 11/21] Now allows abstract dim variable name to be rewritten --- src/codegens/C/codegen.py | 19 ++++++++++--------- src/codegens/base_codegen.py | 31 +++++++++++++++++++------------ src/codegens/matlab/codegen.py | 3 ++- src/codegens/python/codegen.py | 2 ++ src/codes/encoders/c_encoder.py | 6 +++--- src/properties/abstract_dim.py | 5 +++-- src/properties/shape.py | 19 +++++++++++++++++-- 7 files changed, 56 insertions(+), 29 deletions(-) diff --git a/src/codegens/C/codegen.py b/src/codegens/C/codegen.py index 40c236b..3840df0 100644 --- a/src/codegens/C/codegen.py +++ b/src/codegens/C/codegen.py @@ -122,16 +122,16 @@ def c_dimensions(self): self.size_lookup['m'] = self.num_conic + self.num_lps self.size_lookup['n'] = self.num_vars self.size_lookup['p'] = self.num_lineqs - yield "data->p = %d;" % self.num_lineqs - yield "data->m = %d;" % (self.num_conic + self.num_lps) - yield "data->n = %d;" % self.num_vars + yield "data->p = %s;" % self.num_lineqs + yield "data->m = %s;" % (self.num_conic + self.num_lps) + yield "data->n = %s;" % self.num_vars # generator to get cone dimensions def c_cone_sizes(self): num_cone, cone_size = zip(*self.cone_list) - yield "data->l = %d;" % self.num_lps - yield "data->nsoc = %d;" % sum(num_cone) + yield "data->l = %s;" % self.num_lps + yield "data->nsoc = %s;" % sum(num_cone) if num_cone == 0: yield "data->q = NULL;" else: @@ -287,13 +287,13 @@ def stuff_c(self, start, end, expr): # TODO: i shouldn't have to check here.... if expr.isscalar or isinstance(expr, OnesCoeff): tag = ";" else: tag = "[i];" - yield "for(i = 0; i < %d; ++i) data->c[i + %s] = %s%s" % (end-start, start, toC(expr), tag) + yield "for(i = 0; i < %s; ++i) data->c[i + %s] = %s%s" % (end-start, start, toC(expr), tag) def stuff_b(self, start, end, expr): # TODO: i shouldn't have to check here.... if expr.isscalar or isinstance(expr, OnesCoeff): tag = ";" else: tag = "[i];" - yield "for(i = 0; i < %d; ++i) data->b[i + %s] = %s%s" % (end-start, start, toC(expr), tag) + yield "for(i = 0; i < %s; ++i) data->b[i + %s] = %s%s" % (end-start, start, toC(expr), tag) def stuff_h(self, start, end, expr, stride = None): if expr.isscalar: tag = ";" @@ -302,7 +302,7 @@ def stuff_h(self, start, end, expr, stride = None): numel = math.ceil( float(end - start) / stride ) yield "for(i = 0; i < %d; ++i) data->h[%s * i + %s] = %s%s" % (numel, stride, start, toC(expr), tag) else: - yield "for(i = 0; i < %d; ++i) data->h[i + %s] = %s%s" % (end-start, start, toC(expr), tag) + yield "for(i = 0; i < %s; ++i) data->h[i + %s] = %s%s" % (end-start, start, toC(expr), tag) def stuff_matrix(self, matrix, row_start, row_end, col_start, col_end, expr, row_stride): @@ -328,5 +328,6 @@ def stuff_A(self, row_start, row_end, col_start, col_end, expr, row_stride = 1): # but then return this generator return self.stuff_matrix("A", row_start, row_end, col_start, col_end, expr, row_stride) - + def abstractdim_rewriter(self, ad): + return "dims.%s" % ad diff --git a/src/codegens/base_codegen.py b/src/codegens/base_codegen.py index f754d53..355aa1b 100644 --- a/src/codegens/base_codegen.py +++ b/src/codegens/base_codegen.py @@ -106,6 +106,13 @@ def stuff_A(self, row_start, row_end, col_start, col_end, expr, row_stride = Non """ yield "" + @abstractmethod + def abstractdim_rewriter(self, ad): + """ Translate a raw abstract dimension name like 'm' or 'n' into a + name like 'dims.m' or dims('n') + """ + return "%s" % ad + def codegen(self): # create the source code self.prob2socp.create() @@ -132,10 +139,10 @@ def visit_Program(self, node): # create variable ordering # XXX: at this moment, assumes that variables are vectors (not arrays) # varlength contains the vector lengths as ints - self.varlength = {k: v.shape.size() \ + self.varlength = {k: v.shape.size(abstractdim_rewriter=self.abstractdim_rewriter) \ for k,v in node.variables.iteritems() } - self.varlength.update({k: v.shape.size() \ + self.varlength.update({k: v.shape.size(abstractdim_rewriter=self.abstractdim_rewriter) \ for k,v in node.new_variables.iteritems() }) @@ -157,7 +164,7 @@ def visit_Program(self, node): self.functions_return(node) def visit_Variable(self, node): - n = node.shape.size() + n = node.shape.size(abstractdim_rewriter=self.abstractdim_rewriter) k = node.value if n == 1: @@ -212,7 +219,7 @@ def visit_Sum(self, node): arg = self.expr_stack.pop() for k in arg.keys(): - n = node.expr.shape.size() + n = node.expr.shape.size(abstractdim_rewriter=self.abstractdim_rewriter) arg[k] = OnesCoeff(n, ConstantCoeff(1), True) * arg[k] self.expr_stack.append(arg) @@ -263,10 +270,10 @@ def visit_LinearConstraint(self, node): if node.op == '==': start = self.num_lineqs - self.num_lineqs += node.shape.size() + self.num_lineqs += node.shape.size(abstractdim_rewriter=self.abstractdim_rewriter) else: start = self.num_lps - self.num_lps += node.shape.size() + self.num_lps += node.shape.size(abstractdim_rewriter=self.abstractdim_rewriter) self.generic_visit(node) @@ -307,11 +314,11 @@ def visit_SOC(self, node): # we assume linear constraints have already been handled start = [self.num_lps + self.num_conic] - start += [start[-1] + node.right.shape.size()] - cone_length = node.right.shape.size() + start += [start[-1] + node.right.shape.size(abstractdim_rewriter=self.abstractdim_rewriter)] + cone_length = node.right.shape.size(abstractdim_rewriter=self.abstractdim_rewriter) for e in node.left: - start += [start[-1] + e.shape.size()] - cone_length += e.shape.size() + start += [start[-1] + e.shape.size(abstractdim_rewriter=self.abstractdim_rewriter)] + cone_length += e.shape.size(abstractdim_rewriter=self.abstractdim_rewriter) self.num_conic += cone_length self.cone_list.append( (1, str(cone_length)) ) @@ -342,9 +349,9 @@ def visit_SOCProd(self, node): # we assume linear constraints have already been handled start = self.num_lps + self.num_conic stride = node.nargs + 1 - self.num_conic += stride * node.shape.size() + self.num_conic += stride * node.shape.size(abstractdim_rewriter=self.abstractdim_rewriter) - self.cone_list.append( (node.shape.size(), str(node.nargs + 1)) ) + self.cone_list.append( (node.shape.size(abstractdim_rewriter=self.abstractdim_rewriter), str(node.nargs + 1)) ) self.generic_visit(node) diff --git a/src/codegens/matlab/codegen.py b/src/codegens/matlab/codegen.py index 8682666..530a27b 100644 --- a/src/codegens/matlab/codegen.py +++ b/src/codegens/matlab/codegen.py @@ -88,4 +88,5 @@ def stuff_A(self, r0, rend, c0, cend, expr, rstride = 1): def stuff_G(self, r0, rend, c0, cend, expr, rstride = 1): return self.stuff_matrix("G", r0, rend, c0, cend, expr, rstride) - + def abstractdim_rewriter(self, ad): + return "dims.%s" % ad diff --git a/src/codegens/python/codegen.py b/src/codegens/python/codegen.py index 0f6f3ae..b6c46b6 100644 --- a/src/codegens/python/codegen.py +++ b/src/codegens/python/codegen.py @@ -121,3 +121,5 @@ def stuff_A(self, row_start, row_end, col_start, col_end, expr, row_stride = 1): yield "Aj.append(%s)" % toPython(expr.J(col_start)) yield "Av.append(%s)" % toPython(expr.V()) + def abstractdim_rewriter(self, ad): + return "dims['%s']" % ad diff --git a/src/codes/encoders/c_encoder.py b/src/codes/encoders/c_encoder.py index a42fe31..da4f183 100644 --- a/src/codes/encoders/c_encoder.py +++ b/src/codes/encoders/c_encoder.py @@ -46,16 +46,16 @@ def to_str(x): if x.offset == 0 and x.stride == 1: s = "for(i = 0; i < %(matrix)s->nnz; ++i) *%%(ptr)s++ = %(matrix)s->%(ijv)s[i];" else: - s = "for(i = 0; i < %(matrix)s->nnz; ++i) *%%(ptr)s++ = %(offset)d + %(stride)d*(%(matrix)s->%(ijv)s[i]);" + s = "for(i = 0; i < %(matrix)s->nnz; ++i) *%%(ptr)s++ = %(offset)s + %(stride)s*(%(matrix)s->%(ijv)s[i]);" return s % ({'matrix': matrix, 'offset': x.offset, 'stride': x.stride, 'ijv': ijv}) return "for(i = 0; i < %s->nnz; ++i) *%%(ptr)s++ = %s;" % (matrix, x.op % ("%s->%s[i]" % (matrix, ijv))) return to_str def _range(x): if x.stride == 1: - return "for(i = %d; i < %d; ++i) *%%(ptr)s++ = i;" % (x.start, x.end) + return "for(i = %s; i < %s; ++i) *%%(ptr)s++ = i;" % (x.start, x.end) else: - return "for(i = %d; i < %d; i+=%d) *%%(ptr)s++ = i;" % (x.start, x.end, x.stride) + return "for(i = %s; i < %s; i+=%s) *%%(ptr)s++ = i;" % (x.start, x.end, x.stride) def repeat(x): return "for(i = 0; i < %s; ++i) *%%(ptr)s++ = %s;" % (x.n, toC(x.obj)) diff --git a/src/properties/abstract_dim.py b/src/properties/abstract_dim.py index 891213d..460277a 100644 --- a/src/properties/abstract_dim.py +++ b/src/properties/abstract_dim.py @@ -1,10 +1,11 @@ import collections def list_product(l): - """ Take a list and return an AD by multiplying list elements together + """ Take a list and return an AD by multiplying list elements together. """ return reduce(lambda x,y: x * AbstractDim(y), l, AbstractDim(1)) + class AbstractDim(object): """ Implemented by holding a collections.Counter to keep track of terms. @@ -150,4 +151,4 @@ def __rmul__(self, other): return AbstractDim(other) * self if __name__ == "__main__": - print AbstractDim() + print list_product([5, 'a']) diff --git a/src/properties/shape.py b/src/properties/shape.py index 80879df..89f65e8 100644 --- a/src/properties/shape.py +++ b/src/properties/shape.py @@ -86,8 +86,23 @@ def _assign_col(self): def _check_instantiation(self): self.instantiated = all(type(elem) == int for elem in self.dimensions) - def size(self): - return abstract_dim.list_product(self.dimensions) + def size(self, abstractdim_rewriter=None): + """ Returns product of dimensions, wrapped in an AbstractDim object + to handle dimensions that are still abstract, i.e. still string + variable names like 'm' or 'n'. + + Additionally, abstract string dimension names can be rewritten + (usually for consistency with a particular codegen). See + Codegen.abstractdim_rewriter for examples. + """ + dims = self.dimensions + if abstractdim_rewriter: + # Only apply it to dims that are still abstract + def adrw(x): + if isinstance(x, str): return abstractdim_rewriter(x) + return x + dims = map(adrw, dims) + return abstract_dim.list_product(dims) def __str__(self): if isscalar(self): return "Scalar()" From 13796ff13f286f5f8373bcdef298c69be9037e36 Mon Sep 17 00:00:00 2001 From: Peter Li Date: Sun, 15 Sep 2013 19:21:39 -0700 Subject: [PATCH 12/21] Switch back to m,n for CBP example --- examples/cbp.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/examples/cbp.py b/examples/cbp.py index 44152d0..8f252f8 100755 --- a/examples/cbp.py +++ b/examples/cbp.py @@ -22,18 +22,18 @@ p = QCML(debug=True) p.parse(""" - dimensions q r - variable c(r) - variable u(r) - variable v(r) + dimensions m n + variable c(n) + variable u(n) + variable v(n) parameter noise positive - parameter lambda(r) - parameter data(q) - parameter dictc(q,r) - parameter dictu(q,r) - parameter dictv(q,r) - parameter radii(r,r) # diagonal matrix - parameter rctheta(r,r) # diagonal matrix + parameter lambda(n) + parameter data(m) + parameter dictc(m,n) + parameter dictu(m,n) + parameter dictv(m,n) + parameter radii(n,n) # diagonal matrix + parameter rctheta(n,n) # diagonal matrix minimize noise*norm(data - (dictc*c + dictu*u + dictv*v)) + lambda'*c subject to # || (u[i], v[i]) || <= radii_i * c_i From a3eda1a1f7b5a8773b4b07e197c94dd786c6b671 Mon Sep 17 00:00:00 2001 From: Peter Li Date: Sun, 15 Sep 2013 19:21:55 -0700 Subject: [PATCH 13/21] In progress gettings dims argument into prob2socp Mostly done but need to get dims struct into the C header file --- src/codegens/C/codegen.py | 8 +++++--- src/codegens/base_codegen.py | 2 +- src/codegens/matlab/codegen.py | 14 +++++++------- src/codegens/python/codegen.py | 8 ++++---- 4 files changed, 17 insertions(+), 15 deletions(-) diff --git a/src/codegens/C/codegen.py b/src/codegens/C/codegen.py index 3840df0..75e8832 100644 --- a/src/codegens/C/codegen.py +++ b/src/codegens/C/codegen.py @@ -52,10 +52,12 @@ def __init__(self, sparsity_pattern = None, name = "problem"): # functions we are going to generate self.__prob2socp = CFunction("qc_%s2socp" % self.name, - arguments = ["const %s_params * params" % self.name], + arguments = ["const %s_params * params" % self.name, + "const %s_dims * dims" % self.name], ret_type="qc_socp *") self.__socp2prob = CFunction("qc_socp2%s" % self.name, - arguments = ["double * x", "%s_vars * vars" % self.name]) + arguments = ["double * x", "%s_vars * vars" % self.name, + "const %s_dims * dims" % self.name]) # get the paths to the template files template_path = os.path.dirname(__file__) @@ -329,5 +331,5 @@ def stuff_A(self, row_start, row_end, col_start, col_end, expr, row_stride = 1): return self.stuff_matrix("A", row_start, row_end, col_start, col_end, expr, row_stride) def abstractdim_rewriter(self, ad): - return "dims.%s" % ad + return "dims->%s" % ad diff --git a/src/codegens/base_codegen.py b/src/codegens/base_codegen.py index 355aa1b..638f9aa 100644 --- a/src/codegens/base_codegen.py +++ b/src/codegens/base_codegen.py @@ -130,7 +130,7 @@ def pmn(self): yield (self.num_lineqs, self.num_conic + self.num_lps, self.num_vars) @property - def dimsl(self): yield self.num_lps + def conesl(self): yield self.num_lps def visit_Program(self, node): # keep track of original variables diff --git a/src/codegens/matlab/codegen.py b/src/codegens/matlab/codegen.py index 530a27b..7aab8c3 100644 --- a/src/codegens/matlab/codegen.py +++ b/src/codegens/matlab/codegen.py @@ -7,8 +7,8 @@ class MatlabCodegen(Codegen): def __init__(self): super(MatlabCodegen, self).__init__() - self.__prob2socp = MatlabFunction("prob_to_socp", ["params"], ["data"]) - self.__socp2prob = MatlabFunction("socp_to_prob", ["x"], ["vars"]) + self.__prob2socp = MatlabFunction('prob_to_socp', ['params', 'dims'], ['data']) + self.__socp2prob = MatlabFunction('socp_to_prob', ['x', 'dims'], ['vars']) @property def prob2socp(self): return self.__prob2socp @@ -16,7 +16,7 @@ def prob2socp(self): return self.__prob2socp @property def socp2prob(self): return self.__socp2prob - def dimsq(self): + def conesq(self): def cone_tuple_to_str(x): num, sz = x if num == 1: return str(sz) @@ -37,9 +37,9 @@ def functions_setup(self, program_node): self.prob2socp.add_lines("Ai = []; Aj = []; Av = [];") self.prob2socp.newline() - self.prob2socp.add_lines("dims.l = %s;" % l for l in self.dimsl) - self.prob2socp.add_lines("dims.q = [%s];" % q for q in self.dimsq()) - self.prob2socp.add_lines("dims.s = [];") + self.prob2socp.add_lines("cones.l = %s;" % l for l in self.conesl) + self.prob2socp.add_lines("cones.q = [%s];" % q for q in self.conesq()) + self.prob2socp.add_lines("cones.s = [];") def functions_return(self, program_node): self.prob2socp.add_comment('Convert from sparse triplet to column compressed format.') @@ -48,7 +48,7 @@ def functions_return(self, program_node): self.prob2socp.add_lines("G = sparse(Gi+1, Gj+1, Gv, m, n);") self.prob2socp.newline() self.prob2socp.add_comment('Build output') - self.prob2socp.add_lines("data = struct('c', c, 'b', b, 'h', h, 'G', G, 'A', A, 'dims', dims);") + self.prob2socp.add_lines("data = struct('c', c, 'b', b, 'h', h, 'G', G, 'A', A, 'dims', cones);") recover = ( "'%s', x(%s:%s)" % (k, self.varstart[k], self.varstart[k]+self.varlength[k]) diff --git a/src/codegens/python/codegen.py b/src/codegens/python/codegen.py index b6c46b6..e57f9f0 100644 --- a/src/codegens/python/codegen.py +++ b/src/codegens/python/codegen.py @@ -12,8 +12,8 @@ def wrapped_code(self, *args, **kwargs): class PythonCodegen(Codegen): def __init__(self): super(PythonCodegen, self).__init__() - self.__prob2socp = PythonFunction("prob_to_socp", ["params"]) - self.__socp2prob = PythonFunction("socp_to_prob", ["x"]) + self.__prob2socp = PythonFunction('prob_to_socp', ['params', 'dims']) + self.__socp2prob = PythonFunction('socp_to_prob', ['x']) @property def prob2socp(self): @@ -38,7 +38,7 @@ def cone_tuple_to_str(x): cone_list_str = map(cone_tuple_to_str, self.cone_list) cone_list_str = '+'.join(cone_list_str) - yield "dims = {'l': %s, 'q': %s, 's': []}" % (self.num_lps, cone_list_str) + yield "cones = {'l': %s, 'q': %s, 's': []}" % (self.num_lps, cone_list_str) def functions_setup(self, program_node): # add some documentation @@ -79,7 +79,7 @@ def functions_return(self, program_node): self.prob2socp.add_lines("else: G, h = None, None") self.prob2socp.add_lines("if p > 0: A = sp.csc_matrix((Av, np.vstack((Ai, Aj))), (p,n))") self.prob2socp.add_lines("else: A, b = None, None") - self.prob2socp.add_lines("return {'c': c, 'G': G, 'h': h, 'A': A, 'b': b, 'dims': dims}") + self.prob2socp.add_lines("return {'c': c, 'G': G, 'h': h, 'A': A, 'b': b, 'dims': cones}") self.socp2prob.document("recovers the problem variables from the solver variable 'x'") # recover the old variables From 3fb6837887f58b00fa8ad481b5eead2a7fba61f4 Mon Sep 17 00:00:00 2001 From: Peter Li Date: Sun, 15 Sep 2013 20:47:04 -0700 Subject: [PATCH 14/21] Geting C codegen setup with flexdims argument --- src/ast/problems/program.py | 4 ++++ src/codegens/C/codegen.py | 6 ++++++ src/codegens/C/stuff_template.h | 7 ++++++- 3 files changed, 16 insertions(+), 1 deletion(-) diff --git a/src/ast/problems/program.py b/src/ast/problems/program.py index 5691b2c..38c3e3a 100644 --- a/src/ast/problems/program.py +++ b/src/ast/problems/program.py @@ -76,6 +76,10 @@ def add_constraint(self, c): def dimensions(self): return self.__dimensions + @property + def abstract_dims(self): + return filter(lambda x: isinstance(x,str), self.dimensions) + @dimensions.setter def dimensions(self, dims): (v.shape.eval(dims) for v in self.variables.values()) diff --git a/src/codegens/C/codegen.py b/src/codegens/C/codegen.py index 75e8832..d08f081 100644 --- a/src/codegens/C/codegen.py +++ b/src/codegens/C/codegen.py @@ -103,6 +103,7 @@ def codegen(self): 'name': self.name, 'NAME': self.name.upper(), 'params': self.params, + 'dims': self.abstract_dims, 'variables': self.variables, 'prob2socp': self.__prob2socp.source, 'socp2prob': self.__socp2prob.source, @@ -150,6 +151,10 @@ def c_cone_sizes(self): def c_params(self, program_node): return ["%s%s %s;" % (self.indent, shape_to_c_type(v),k) for (k,v) in program_node.parameters.iteritems()] + # function to get abstract dims + def c_dims(self, program_node): + return ["%sint %s;" % (self.indent, k) for k in program_node.abstract_dims] + # function to get variables def c_variables(self, program_node): return ["%s%s %s;" % (self.indent, shape_to_c_type(v),k) for (k,v) in program_node.variables.iteritems()] @@ -225,6 +230,7 @@ def functions_setup(self, program_node): self.prob2socp.newline() self.params = '\n'.join(self.c_params(program_node)) + self.abstract_dims = '\n'.join(self.c_dims(program_node)) self.variables = '\n'.join(self.c_variables(program_node)) self.prob2socp.add_comment("local variables") diff --git a/src/codegens/C/stuff_template.h b/src/codegens/C/stuff_template.h index cd8a7bb..6c204e2 100644 --- a/src/codegens/C/stuff_template.h +++ b/src/codegens/C/stuff_template.h @@ -15,6 +15,11 @@ typedef struct params { %(params)s } %(name)s_params; +/* the input dimensions struct */ +typedef struct dims { +%(dims)s +} %(name)s_dims; + /* the solution struct * users are responsible for keep track of the variable lengths */ @@ -32,4 +37,4 @@ typedef struct sol { */ %(socp2prob_prototype)s; -#endif // __%(NAME)s_H__ \ No newline at end of file +#endif // __%(NAME)s_H__ From e5c34d7c1917644bfcba2229194fdba1fb547c78 Mon Sep 17 00:00:00 2001 From: Peter Li Date: Sun, 15 Sep 2013 20:50:32 -0700 Subject: [PATCH 15/21] Didn't mean to include this --- examples/cbp2.py | 74 ------------------------------------------------ 1 file changed, 74 deletions(-) delete mode 100755 examples/cbp2.py diff --git a/examples/cbp2.py b/examples/cbp2.py deleted file mode 100755 index a05f733..0000000 --- a/examples/cbp2.py +++ /dev/null @@ -1,74 +0,0 @@ -#!/usr/bin/env python -""" Continuous basis pursuit example. - - TODO: (PHLI) fill in reference to Eero's paper - See .... -""" - -from argparse import ArgumentParser -from qcml import QCML - -if __name__ == "__main__": - parser = ArgumentParser(description='Continuous Basis Pursuit QCML example') - parser.add_argument('m', type=int, help='Length of waveform (samples)') - parser.add_argument('n', type=int, help='Number of templates in dictionary') - parser.add_argument('-c', '--codegen', help='Codegen type to use (python, matlab, or C; default python)', default='python') - args = parser.parse_args() - n, m = (args.n, args.m) - - print "Running CBP example...." - - # TODO: takeaways from this example: "diag" constructor? - - p = QCML(debug=True) - p.parse(""" - dimensions m n - variable c(n) - variable u(n) - variable v(n) - parameter noise positive - parameter lambda(n) - parameter data(m) - parameter dictc(m,n) - parameter dictu(m,n) - parameter dictv(m,n) - parameter radii(n,n) # diagonal matrix - parameter rctheta(n,n) # diagonal matrix - minimize noise*norm(data - (dictc*c + dictu*u + dictv*v)) + lambda'*c - subject to - # || (u[i], v[i]) || <= radii_i * c_i - # norm([u_i v_i]) <= radii_i*c_i - # norm(x,y) applies norm across rows of the matrix [x y] - norm(u,v) <= radii*c - - # rctheta is going to be a diagonal matrix - # rctheta[i]*c[i] <= u[i] implemented with rctheta a diag matrix - rctheta*c <= u - - c <= 1.5 - """) - - # More natural formulation would be: - # minimize 1/(sqrt(2)*noisesigma) * (data - (dictc*c + dictu(u + dictv*v)) + lambda'*c) - # subject to - # sqrt(u.^2 + v.^2) <= radii.*c - # radii.*cos(theta) <= u - - raw_input("press ENTER to canonicalize....") - p.canonicalize() - - raw_input("press ENTER to generate code....") - p.dims = {'n': n, 'm': m} - p.codegen(args.codegen) - - raw_input("press ENTER for raw code....") - print p.prob2socp.source - print p.socp2prob.source - - #socp_data = p.prob2socp(params=locals()) - #import ecos - #sol = ecos.ecos(**socp_data) - #my_vars = p.socp2prob(sol['x']) - #pr.disable() - #ps = pstats.Stats(pr) - #ps.sort_stats('cumulative').print_stats(.5) From 25f881c3dc74d7f2bdebaf5c9ca4d1cd2424414f Mon Sep 17 00:00:00 2001 From: Peter Li Date: Sun, 15 Sep 2013 20:55:44 -0700 Subject: [PATCH 16/21] Finish changing back to m,n for CBP example --- examples/cbp.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/examples/cbp.py b/examples/cbp.py index 8f252f8..019bd77 100755 --- a/examples/cbp.py +++ b/examples/cbp.py @@ -10,11 +10,11 @@ if __name__ == "__main__": parser = ArgumentParser(description='Continuous Basis Pursuit QCML example') - parser.add_argument('-q', type=int, help='Length of waveform (samples)') - parser.add_argument('-r', type=int, help='Number of templates in dictionary') + parser.add_argument('-m', type=int, help='Length of waveform (samples)') + parser.add_argument('-n', type=int, help='Number of templates in dictionary') parser.add_argument('-c', '--codegen', help='Codegen type to use (python, matlab, or C; default python)', default='python') args = parser.parse_args() - q, r = (args.q, args.r) + m, n = (args.m, args.n) print "Running CBP example...." @@ -56,7 +56,7 @@ p.canonicalize() raw_input("press ENTER to generate code....") - if q and r: p.dims = {'q': q, 'r': r} + if m and n: p.dims = {'m': m, 'n': n} p.codegen(args.codegen) raw_input("press ENTER for raw code....") From e063fedee2726d0f1c8b42e8f6f8c825cb7cabdc Mon Sep 17 00:00:00 2001 From: Peter Li Date: Sun, 15 Sep 2013 22:40:16 -0700 Subject: [PATCH 17/21] Default dims={} so works with 1 arg for concrete dim problems --- src/codegens/python/codegen.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/codegens/python/codegen.py b/src/codegens/python/codegen.py index e57f9f0..76cdfb3 100644 --- a/src/codegens/python/codegen.py +++ b/src/codegens/python/codegen.py @@ -12,7 +12,7 @@ def wrapped_code(self, *args, **kwargs): class PythonCodegen(Codegen): def __init__(self): super(PythonCodegen, self).__init__() - self.__prob2socp = PythonFunction('prob_to_socp', ['params', 'dims']) + self.__prob2socp = PythonFunction('prob_to_socp', ['params', 'dims={}']) self.__socp2prob = PythonFunction('socp_to_prob', ['x']) @property From eb96b98888ce57e2d960c3a158407dcf840dcc47 Mon Sep 17 00:00:00 2001 From: Chinasaur Date: Sun, 15 Sep 2013 22:51:35 -0700 Subject: [PATCH 18/21] Bugfix; socp2prob needed dims arg too --- src/codegens/python/codegen.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/codegens/python/codegen.py b/src/codegens/python/codegen.py index 76cdfb3..36b227e 100644 --- a/src/codegens/python/codegen.py +++ b/src/codegens/python/codegen.py @@ -13,7 +13,7 @@ class PythonCodegen(Codegen): def __init__(self): super(PythonCodegen, self).__init__() self.__prob2socp = PythonFunction('prob_to_socp', ['params', 'dims={}']) - self.__socp2prob = PythonFunction('socp_to_prob', ['x']) + self.__socp2prob = PythonFunction('socp_to_prob', ['x', 'dims={}']) @property def prob2socp(self): From c2a7c46e41eeedb28525b7ebed785ff2c76b32ab Mon Sep 17 00:00:00 2001 From: Chinasaur Date: Sun, 15 Sep 2013 23:11:12 -0700 Subject: [PATCH 19/21] Bugfix; needed to include abstract_dims in C_Codegen __init__ --- src/codegens/C/codegen.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/codegens/C/codegen.py b/src/codegens/C/codegen.py index d08f081..6cef2b8 100644 --- a/src/codegens/C/codegen.py +++ b/src/codegens/C/codegen.py @@ -76,6 +76,7 @@ def __init__(self, sparsity_pattern = None, name = "problem"): # parameters and variables in the optimization problem self.params = "" + self.abstract_dims = "" self.variables = "" self.indent = self.__prob2socp.indent # set our indent spacing @@ -338,4 +339,3 @@ def stuff_A(self, row_start, row_end, col_start, col_end, expr, row_stride = 1): def abstractdim_rewriter(self, ad): return "dims->%s" % ad - From 8de606dbd7e95b20b916452b7a094116f7def1ec Mon Sep 17 00:00:00 2001 From: Chinasaur Date: Sun, 15 Sep 2013 23:11:38 -0700 Subject: [PATCH 20/21] Updated test to have dims={} in empty python codegen --- src/test/test_codegen.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test/test_codegen.py b/src/test/test_codegen.py index 6a658c5..43566e3 100644 --- a/src/test/test_codegen.py +++ b/src/test/test_codegen.py @@ -30,10 +30,10 @@ def test_codegen(): yield codegen, gen def test_empty_python_prob2socp(): - assert python.prob2socp.source == "def prob_to_socp(params):\n pass" + assert python.prob2socp.source == "def prob_to_socp(params, dims={}):\n pass" def test_empty_python_socp2prob(): - assert python.socp2prob.source == "def socp_to_prob(x):\n pass" + assert python.socp2prob.source == "def socp_to_prob(x, dims={}):\n pass" c_files = [ "test_problem", From 720dfa20e9b9a80a7a9f1396f49a16ce56bce790 Mon Sep 17 00:00:00 2001 From: Chinasaur Date: Sun, 15 Sep 2013 23:12:37 -0700 Subject: [PATCH 21/21] Add a newline --- examples/cbp.py | 1 + 1 file changed, 1 insertion(+) diff --git a/examples/cbp.py b/examples/cbp.py index 019bd77..e6afc7e 100755 --- a/examples/cbp.py +++ b/examples/cbp.py @@ -61,6 +61,7 @@ raw_input("press ENTER for raw code....") print p.prob2socp.source + print "\n" print p.socp2prob.source #socp_data = p.prob2socp(params=locals())