NNNL fixes

py/core.py

@@ -4248,6 +4248,9 @@ def _del_casewise_d_logsums(self) -> "void":
     def top_logsums_out_currently_valid(self) -> "bool":
         return _core.Model2_top_logsums_out_currently_valid(self)
 
+    def _setUp_NNNL_host(self, ncases: 'unsigned int const &') -> "void":
+        return _core.Model2__setUp_NNNL_host(self, ncases)
+
     def _parameter_report(self, other1: 'etk::ndarray const *'=None, other2: 'etk::ndarray const *'=None) -> "std::string":
         return _core.Model2__parameter_report(self, other1, other2)
     _LL_null = _swig_property(_core.Model2__LL_null_get, _core.Model2__LL_null_set)

py/dt/__init__.py

@@ -932,8 +932,11 @@ def array_avail(self, *, var=None, dtype=numpy.bool_, **kwargs):
 			return self.array_idca('1', dtype=dtype, **kwargs)
 		if isinstance(self.idca._avail_, (_tb.Group,GroupNode)):
 			##!stacktuple = self.idca._avail_._v_attrs.stack
-			stacktuple = self.from_vault('stack._avail_')
-			return numpy.expand_dims(self.array_idco(*stacktuple, dtype=dtype, **kwargs), axis=-1)
+			if not self.in_vault('stack._avail_'):
+				return self.array_idca('_avail_', dtype=dtype, **kwargs)
+			else:
+				stacktuple = self.from_vault('stack._avail_')
+				return numpy.expand_dims(self.array_idco(*stacktuple, dtype=dtype, **kwargs), axis=-1)
 		else:
 			return self.array_idca('_avail_', dtype=dtype, **kwargs)
 
@@ -2543,6 +2546,18 @@ def new_blank_idco(self, name, dtype=None, overwrite=False, title=None):
 		zer = numpy.zeros(self.nAllCases(), dtype=dtype or numpy.float64)
 		return self.new_idco_from_array(name, zer, overwrite=overwrite, title=title)
 
+	def new_seqential_idco(self, name, dtype=None, overwrite=False, title=None):
+		zer = numpy.arange(self.nAllCases(), dtype=dtype or numpy.float64)
+		return self.new_idco_from_array(name, zer, overwrite=overwrite, title=title)
+
+	def new_random_idco(self, name, overwrite=False, title=None, seed=None, generator=None):
+		if seed is not None:
+			numpy.random.seed(seed)
+		if generator is None:
+			generator = lambda x: numpy.random.random(x)
+		zer = generator(self.nAllCases())
+		return self.new_idco_from_array(name, zer, overwrite=overwrite, title=title)
+
 	def new_idco_from_array(self, name, arr, *, overwrite=False, original_source=None, rel_original_source=True, title=None, dictionary=None):
 		"""Create a new :ref:`idco` variable.
 		
@@ -2984,7 +2999,7 @@ def new_idco_from_keyed_array(self, name, arr_val, arr_index, title=None):
 			raise TypeError("arr_index invalid type ({})".format(str(type(arr_index))))
 
 
-	def new_idca_from_keyed_array(self, name, arr_val, arr_index, transpose_values=False):
+	def new_idca_from_keyed_array(self, name, arr_val, arr_index=None, transpose_values=False, overwrite=False):
 		"""Create a new :ref:`idca` variable.
 		
 		Creating a new variable in the data might be convenient in some instances.
@@ -2998,17 +3013,21 @@ def new_idca_from_keyed_array(self, name, arr_val, arr_index, transpose_values=F
 		arr_val : ndarray
 			An array to add as the new variable _values_.  The 2nd dimension must match the
 			number of alternatives.
-		arr_index : ndarray or pytable node
+		arr_index : ndarray or pytable node or None
 			An array to add as the new variable _index_.  It must be 1 dimension and must match the
-			number of caseids.
+			number of caseids.  If None, then arr_val can be 1d and is interpreted as a row vector.
 			
 		Raises
 		-----
 		_tb.exceptions.NodeError
 			If a variable of the same name already exists.
 		"""
-		if self.h5caseids.shape[0] != arr_index.shape[0]:
+		if arr_index is not None and self.h5caseids.shape[0] != arr_index.shape[0]:
 			raise TypeError("new idca array must have shape with {!s} cases, the input array has {!s} cases".format(self.h5caseids.shape[0], arr.shape[0]))
+		if arr_index is None and len(arr_val.shape)==1:
+			arr_val = arr_val.reshape(1,-1)
+		if overwrite:
+			self.delete_data(name)
 		newgrp = self.h5f.create_group(self.idca._v_node, name)
 		self.h5f.create_carray(newgrp, '_values_', obj=arr_val)
 		if transpose_values:
@@ -3017,6 +3036,8 @@ def new_idca_from_keyed_array(self, name, arr_val, arr_index, transpose_values=F
 			self.h5f.create_carray(newgrp, '_index_', obj=arr_index)
 		elif isinstance(arr_index, _tb.array.Array):
 			self.h5f.create_soft_link(newgrp, '_index_', arr_index)
+		elif arr_index is None:
+			self.h5f.create_carray(newgrp, '_index_', obj=numpy.zeros(self.nAllCases(), dtype=numpy.int8))
 		else:
 			raise TypeError("arr_index invalid type ({})".format(str(type(arr_index))))
 

py/dt/analyze.py

@@ -27,7 +27,10 @@ def seer(self, big=False):
 
 
 
-def look_site(self, directory=None, screen="None"):
+def look_site(self, directory=None, screen="None", clear_cache=False):
+
+	if clear_cache:
+		self.clear_look_cache()
 
 	if directory is None:
 		raise NotImplementedError('give a directory')

py/dt/groupnode.py

@@ -4,7 +4,8 @@
 import types
 import pandas
 import os
-from ..util.naming import make_valid_identifier
+import warnings
+from ..util.naming import make_valid_identifier, NotAPythonIdentifier
 
 #class GroupNode(tables.group.Group):
 #	def __new__(cls, parentnode, name=None, *arg, **kwarg):
@@ -209,7 +210,6 @@ def __contains__(self, arg):
 			try:
 				extern_deref = _pytables_link_dereference(self._v_node._v_children["_extern_{}".format(extern_n)])
 			except OSError as err:
-				import warnings
 				warnings.warn(str(err))
 			else:
 				if arg in _get_children_including_extern(extern_deref):
@@ -282,7 +282,7 @@ def add_external_data(self, link):
 		return self._v_file.create_external_link(self._v_node, '_extern_{}'.format(extern_n), link)
 
 
-	def add_external_omx(self, omx_filename, rowindexnode, prefix="", n_alts=-1, n_lookup=-1, absolute_path=False, local_rowindexnode=None):
+	def add_external_omx(self, omx_filename, rowindexnode, prefix="", n_alts=-1, n_lookup=-1, absolute_path=False, local_rowindexnode=None, suppress_identifier_warning=False):
 		'''
 		Add an external linkage from this group to the values in an OMX file.
 		
@@ -339,7 +339,6 @@ def rowindexnode_():
 				try:
 					vgrp = self._v_file.create_group(self._v_node, prefix+vname)
 				except tables.exceptions.NodeError:
-					import warnings
 					warnings.warn('the name "{}" already exists'.format(prefix+vname))
 				else:
 					self._v_file.create_soft_link(vgrp, '_index_', rowindexnode_())
@@ -352,11 +351,13 @@ def rowindexnode_():
 					# The constructed index is len(caseids) but all zeros.
 					# Each values from the index plucks the entire lookup vector.
 					# The resulting pseudoarray is shape (nCases,nAlts)
-					full_lname = make_valid_identifier(prefix+lname)
+					with warnings.catch_warnings():
+						if suppress_identifier_warning:
+							warnings.filterwarnings('ignore', category=NotAPythonIdentifier)
+						full_lname = make_valid_identifier(prefix+lname)
 					try:
 						vgrp = self._v_file.create_group(self._v_node, full_lname)
 					except tables.exceptions.NodeError:
-						import warnings
 						warnings.warn('the name "{}" already exists'.format(full_lname))
 					else:
 						self._v_file.create_carray(vgrp, '_index_', shape=_pytables_link_dereference(self._v_file.root.larch.caseids).shape, atom=tables.Int32Atom()) # zeros
@@ -372,14 +373,12 @@ def rowindexnode_():
 					try:
 						vgrp = self._v_file.create_group(self._v_node, full_lname)
 					except tables.exceptions.NodeError:
-						import warnings
 						warnings.warn('the name "{}" already exists'.format(full_lname))
 					else:
 						self._v_file.create_soft_link(vgrp, '_index_', rowindexnode)
 						self._v_file.create_external_link(vgrp, '_values_', omx_filename+":/lookup/"+lname)
 						anything_linked = True
 		if not anything_linked:
-			import warnings
 			from ..omx import OMX
 			try:
 				omx_repr = repr(OMX(omx_filename))

py/model.py

@@ -84,6 +84,57 @@ def px(self, n):
 			raise LarchError("not implemented")
 		return ModelParameter(self, n)
 
+	def stash_parameters(self, ticket='generic', stashdir=None):
+		if ticket=='generic' and self.title:
+			ticket = self.title
+		import json
+		if stashdir is None:
+			try:
+				import appdirs
+			except ImportError:
+				raise TypeError("no stashdir given and appdirs not available")
+			else:
+				stashdir = appdirs.user_cache_dir('larch')
+		from .util.filemanager import next_stack
+		os.makedirs(stashdir, exist_ok=True)
+		filename = next_stack(os.path.join(stashdir, 'ticket_{}.json'.format(ticket)))
+		parms={}
+		for p in self:
+			parms[p.name] = p.value
+		with open(filename, 'a') as s:
+			json.dump(parms, s)
+
+	def unstash_parameters(self, ticket='generic', stashdir=None):
+		if ticket=='generic' and self.title:
+			ticket = self.title
+		import json
+		if stashdir is None:
+			try:
+				import appdirs
+			except ImportError:
+				raise TypeError("no stashdir given and appdirs not available")
+			else:
+				stashdir = appdirs.user_cache_dir('larch')
+		from .util.filemanager import latest_matching
+		filename = latest_matching(os.path.join(stashdir, 'ticket_{}.*.json'.format(ticket)))
+		if filename is None:
+			return
+		parms = {}
+		with open(filename, 'r') as s:
+			parms = json.load(s)
+		for p in self:
+			if p.name in parms:
+				val = parms[p.name]
+				if p.holdfast:
+					pass
+				elif val < p.min_value:
+					p.value = p.min_value
+				elif val > p.max_value:
+					p.value = p.max_value
+				else:
+					p.value = val
+
+
 
 	def add_parameter(self, name, **kwargs):
 		if isinstance(name, ModelParameter):

py/nnnl.py

@@ -193,8 +193,7 @@ def _specific_warning_notes(self):
 	def setUp(self, *args, **kwargs):
 		self.sub_weight = {}
 		self.total_weight = 0
-		self.sub_ncases = {}
-		self.total_ncases = 0
+		self._ncases = 0
 		for seg_descrip,submodel in self.sub_model.items():
 			if self.logger():
 				self.logger().log(30, "setUp:{!s}".format(seg_descrip))
@@ -230,13 +229,12 @@ def setUp(self, *args, **kwargs):
 				submodel.provision(prov)
 
 			if submodel.df.nCases()==0:
-				submodel.sub_ncases[seg_descrip] = 0
+				submodel._ncases = 0
 				submodel.sub_weight[seg_descrip] = 0
 				continue
 			self.sub_weight[seg_descrip] = partwgt = submodel.Data("Weight").sum()
 			self.total_weight += partwgt
-			self.sub_ncases[seg_descrip] = partncase = submodel.nCases()
-			self.total_ncases += partncase
+			self._ncases = partncase = submodel.nCases()
 		# circle back to update choice and avail of upper level
 		m0 =self.sub_model[self.root_id]
 		self.m0slots = { subkey:slot for slot,subkey in enumerate(m0.alternative_codes()) }
@@ -245,6 +243,7 @@ def setUp(self, *args, **kwargs):
 			m0.dataedit.choice[:,self.m0slots[key]] = (self.sub_model[key].data.choice * self.sub_model[key].data.avail).sum(1)
 			# set lower level models to feed logsum up
 			self.sub_model[key].top_logsums_out = m0.dataedit.utilityco[:,self.m0slots[key]]
+		self._setUp_NNNL_host(self._ncases)
 
 	def loglike(self, *args, cached=False):
 		fun = 0
@@ -303,11 +302,13 @@ def d_logsums_many(self, *args, cached=False):
 
 
 	def probability_roll_up(self):
+		if self.work.probability.shape[1] != self.df.nAlts() or self.work.probability.shape[0] != self.df.nCases():
+			self._setUp_NNNL_host(self._ncases or self.df.nCases())
 		m0 = self.sub_model[self.root_id]
 		for slot,nestcode in enumerate(m0.alternative_codes()):
 			t = m0.work.probability[:,slot]
 			for subslot, altcode in enumerate(self.sub_model[nestcode].alternative_codes()):
-				topslot = self.df._alternative_slot(altcode)
+				topslot = int(self.df._alternative_slot(altcode))
 				self.work.probability[:,topslot] = self.sub_model[nestcode].work.probability[:,subslot] * t
 
 	def d_loglike(self, *args, cached=False):
@@ -411,4 +412,9 @@ def logger(self, *args, **kwargs):
 			args = ('NNNL',)
 		return super().logger(*args, **kwargs)
 
+	def maximize_loglike(self, *args, **kwargs):
+		super().maximize_loglike(*args, **kwargs)
+		self.probability_roll_up()
 
+
+

py/util/naming.py

@@ -3,6 +3,12 @@
 import keyword
 import re
 
+
+class NotAPythonIdentifier(Warning):
+	pass
+
+
+
 def make_valid_identifier(x):
 	x = str(x)
 	if keyword.iskeyword(x):
@@ -15,7 +21,7 @@ def make_valid_identifier(x):
 	if not y.isidentifier():
 		y = "_"+y
 	if y!=x:
-		warnings.warn("name {0} is not a valid python identifier, converting to {1}".format(x,y), stacklevel=2)
+		warnings.warn("name {0} is not a valid python identifier, converting to {1}".format(x,y), stacklevel=2, category=NotAPythonIdentifier)
 	return y
 
 def parenthize(x, signs_qualify=False):
@@ -39,4 +45,4 @@ def parenthize(x, signs_qualify=False):
 		if numeric.search(x):
 			return x
 		return "({})".format(x)
-	return x
+	return x

py/util/optimize/__init__.py

@@ -242,7 +242,7 @@ def _build_constraints(model, ignore_problems=False, include_bounds=True):
 		constraints = constraints + tuple(model.network_based_constraints())
 	return constraints
 
-def maximize_loglike(model, *arg, ctol=1e-6, options={}, metaoptions=None, two_stage_constraints=False, pre_bhhh=0, cache_data=False, sessionlog=False, sourcecode=None):
+def maximize_loglike(model, *arg, ctol=1e-6, options={}, metaoptions=None, two_stage_constraints=False, pre_bhhh=0, cache_data=False, sessionlog=False, sourcecode=None, stash='generic'):
 	"""
 	Maximize the log likelihood of the model.
 	
@@ -457,6 +457,9 @@ def maximize_loglike(model, *arg, ctol=1e-6, options={}, metaoptions=None, two_s
 		if specific_warning_note:
 			r.stats.write(specific_warning_note)
 
+	if stash:
+		model.stash_parameters(ticket=stash)
+
 	return r
 
 

py/util/plotting.py

@@ -720,6 +720,7 @@ def svg_computed_factor_figure_with_derivative(m, y_funcs,
 			y.append( yf.func(x,m) )
 			y_labels.append( yf.label )
 		fig = plt.figure(figsize=figsize)
+		fig.set_tight_layout(False)
 		ax = plt.subplot(121)
 		ax.set_xlim(min_x,max_x)
 		if logscale_x:
@@ -962,7 +963,7 @@ def svg_validation_latlong(mod, lat, lon, extent=None, figsize=(6.0,10), show_di
 							gridsize=60, headfont='Roboto Slab', textfont='Roboto',
 							colormap='rainbow', tight_layout=True,
 							headerlevel=2, header=None, short_header=None,
-							colormin=None, colormax=None):
+							colormin=None, colormax=None, omit_zero_zero=True):
 	"""
 	A validation mapset for destination choice and similar models.
 	
@@ -1025,6 +1026,13 @@ def next_subplot(title=None, axisbg=None, ticks_off=True):
 
 	next_subplot.plot_n = 1
 
+	if omit_zero_zero:
+		retain = ~((lon==0) & (lat==0))
+		lon = lon[retain]
+		lat = lat[retain]
+		ch_0 = ch_0[retain]
+		pr_0 = pr_0[retain]
+
 	ax1 = next_subplot('Observed')
 	hb1 = ax1.hexbin(lon, lat, C=ch_0, gridsize=gridsize, xscale='linear',
 					 yscale='linear', bins=None, extent=extent, cmap=colormap, 

src/model/elm_model2.h

@@ -508,6 +508,11 @@ namespace elm {
 
 		bool top_logsums_out_currently_valid() const;
 
+
+		void _setUp_NNNL_host(const unsigned& ncases);
+
+
+
 #ifndef SWIG
 
 		void top_logsums_out_recalculated();

src/model/elm_model2_nl.cpp

@@ -46,6 +46,12 @@ ComponentGraphDNA elm::Model2::Input_Graph()
 }
 
 
+void elm::Model2::_setUp_NNNL_host(const unsigned& ncases)
+{
+	Probability.resize(ncases,Xylem.n_elemental());
+}
+
+
 void elm::Model2::_setUp_NL()
 {
 	INFO(msg)<< "Setting up NL model..." ;