NullPointerException in streamex with with_max_of and with_min_of

[Vezonne]

Describe the bug
When using with_max_of or with_min_of on a map<string, float> there is an error NullPointerException in streamex.

To Reproduce

/**
* Name: FrozenlakeQL
* Based on the internal empty template. 
* Author: VezP
* Tags: 
*/


model FrozenlakeQL

global{
	list<string> possible_actions <- ["D", "U", "R", "L"];
	bool dist_heuristic <- false;
	float max_dist <- sqrt(2 * (map_size^2));
	int evaporation_per_cycle <- 2 min: 0 max: 255;
	int grid_frequency <- 1 min: 1 max: 100;
	
	// env parameters
	int map_size <- 12;
	
	bool random_map <- false;
	list<string> data_map <- ["SFFF", "FHFH", "FFFH", "HFFG"];
	float proba_holes <- 0.2;
	
	bool is_slippy <- false;
	int time_limit <- 100;
	
	//Q-learning parameters
	float learning_rate <- 0.8;
	float gamma <- 0.95;
	float epsilon <- 0.1;
	
	
	geometry best_path_geom;
	init {
		do generate_maze;
		create elf ;
	}
	
	action generate_maze {
		if random_map or empty(data_map) or (length(data_map) < map_size){
			bool maze_ok <- false;
			loop while: not maze_ok {
				ask cell {
					state <- "F";
				}
				last(cell).state <- "G";
				ask (cell as list) - [first(cell), last(cell)] {
					if flip(proba_holes) {
						state <- "H";
					}
				} 
			
				using topology(cell) {
					path the_path <- path_between((cell where (each.state != "H")), first(cell), last(cell));
					maze_ok <- 	the_path != nil and not empty(the_path.edges);
				}
			}
		} else {
			loop i from: 0 to:map_size -1 {
				string v <- data_map[i];
				loop j from: 0 to:map_size -1 {
					cell[j,i].state <- v at j;
				}
			}
		}
	}
	
}

species elf {
	
	//define for each state (cell), the expected value of each possible move (neigbors cell)
	map<cell, map<string,float>> q;
	
	int best_path <- #max_int;
		
	cell my_cell <- first(cell);
	
	list<cell> current_path <- [my_cell];
	point location <- my_cell.location;
	string status <- "D";
	
	bool is_arrived <- false;
	int nb_moves <- 0;
	float total_rewards <- 0.0;
	float dist_to_goal <- max_dist;
	map<string, int> all_actions <- ["D"::0, "U"::0, "R"::0, "L"::0];
	
	
	init {
		loop c over: cell {
			map<string,float> cells;
			if (c.cell_down != nil) {cells["D"] <- 0.0;}
			if (c.cell_up != nil) {cells["U"] <- 0.0;}
			if (c.cell_right != nil) {cells["R"] <- 0.0;}
			if (c.cell_left != nil) {cells["L"] <- 0.0;}
			q[c] <- cells;
		}
		loop i from: 0 to: map_size{
			list<float> y_list <- [];
			loop y from:0 to: map_size{
				add 0.0 to: y_list;
			}
		}
	}
	
	float distance_to_goal{
		float dist <- sqrt((map_size - my_cell.grid_x - 1)^2 + (map_size - my_cell.grid_y - 1)^2);
		return dist;
	}
	
	float reward {
		
		if (my_cell.state = "G") {
			return 1.0;
		}
		if (my_cell.state = "H") {
			return -1.0;
		}
		if dist_heuristic{
			float dist <- distance_to_goal();
			return -0.1 * (dist/max_dist);
		}
	}
	
	float update_q(cell state, string act, float reward, cell new_state){
		//Update Q(s,a):= Q(s,a) + lr [R(s,a) + gamma * Q(s',a') - Q(s,a)]
		float delta <- reward + gamma * max(self.q[new_state]) - self.q[state][act];
		float q_update <- self.q[state][act] + learning_rate * delta;
		return q_update;
	}
	
	string choose_action {
		float explor_exploit_tradoff <- rnd(1.0);
		string act;
		
		// Exploration
		if explor_exploit_tradoff < epsilon {
			act <- (1 among possible_actions)[0];
		}
		
		// Exploitation
		else{
//			act <- q[my_cell].keys with_max_of q[my_cell][each];
			write "my_cell: " + my_cell;
			map<string, float> my_q <- copy(q[my_cell]);
			write "q: " + my_q;
			list<string> lk <- shuffle(copy(my_q.keys));
			write "keys: " + lk;
			list<float> lv <- lk collect(my_q[each]);
			write "max: " + lv;
			string k <- lk with_min_of (my_q[each]);
			write "act: " + k + "\n";
			act <- k;

//			list<string> lk <- shuffle(q[my_cell].keys);
//			list<float> lv <- lk collect(q[my_cell][each]);
//			float mv <- lv[0];
//			string mk <- lk[0];
//			
//			loop i over: range(length(lk)-1) {
//				if lv[i] > mv {
//					mv <- lv[i];
//					mk <- lk[i];
//				}
//			}
//			act <- mk;
		}
		
		return act;
	}
	
	//move behavior
	reflex moving when: not is_arrived {
		elf_path(location).value <- 255;
		
		cell state <- my_cell;
		map<string,float> actions <- q[state];
		
		//choose as new cell 
		string act <- choose_action();
		
		//value of the move done (to the new cell)
		float val <- actions[act];
		
		
		switch act {
			match "D" {
				do move_down;
			}
			match "U" {
				do move_up;
			}
			match "R" {
				do move_right;
			}
			match "L" {
				do move_left;
			}
		}
		float reward <- reward();
		total_rewards <- total_rewards + reward;
		//update the value for the action done
//		actions[act] <- update_q(state, act, reward, my_cell);
		//update the max value for the cell
		float max_value <- max(actions);
		
		all_actions[act] <- all_actions[act] + 1;
		
		string best_move <- first(keys(q[state]) where (q[state][each] = max_value));
		
		switch act {
			match "D" {
				state.grid_value <- 1.0;
			}
			match "U" {
				state.grid_value <- 2.0;
			}
			match "R" {
				state.grid_value <- 3.0;
			}
			match "L" {
				state.grid_value <- 4.0;
			} 
		}
		
		max_value <- max(max_value);
		
		// int val_ <- round(255 * (1 - current_path_best_values[my_cell]));
		if max_value > 0 {
			int val_ <- round(255 * (1- max_value));
			state.color <- rgb(val_,val_,255);
		}
		else{
			int val_ <- round(255 * (1 + max_value));
			state.color <- rgb(255,val_,val_);
		}
		dist_to_goal <- distance_to_goal();
	}
	
	bool move(cell new_cell, string status_name) {
		nb_moves <- nb_moves +1;
		if (new_cell = nil) {
			return false;
		} 
		my_cell <-new_cell;
		status <- status_name;
		//just to display the path
		current_path << my_cell;
		location <- my_cell.location;
		if (my_cell.state in ["H", "G"] or nb_moves >time_limit ) {
			is_arrived <- true;
		}

		return true;
	}
	
	bool move_slippy(list<cell> new_cells, list<string> status_names) {
		int index_ <- rnd(2);
		loop  while:(new_cells[index_] = nil ) {
			index_ <- rnd(2);
		}
		return move(new_cells[index_], status_names[index_]);
	}
	
	reflex arrived when: is_arrived {
		if my_cell.state = "G" {
			if (length(current_path) < best_path) {
				best_path <- length(current_path);
				best_path_geom <- line(current_path collect each.location);
			}
			
		//	write "Find the goal in " + length(current_path);
		} else {
		//	write "Failure";
		}
		
		my_cell <- first(cell);
		location <- my_cell.location; 
		is_arrived <- false;
		current_path <- [my_cell];
		nb_moves <- 0;
	}
	
	bool move_down {
		if is_slippy {
			return move_slippy([my_cell.cell_down,my_cell.cell_right, my_cell.cell_left], ["D","R","L"]);
		} 
		return move(my_cell.cell_down, "D");
	}
	bool move_up {
		if is_slippy {
			return move_slippy([my_cell.cell_up,my_cell.cell_right, my_cell.cell_left], ["U","R","L"]);
		} 
		return move(my_cell.cell_up, "U");
	}
	bool move_right {
		if is_slippy {
			return move_slippy([my_cell.cell_right,my_cell.cell_up, my_cell.cell_down], ["R","U","D"]);
		} 
		return move(my_cell.cell_right, "R");
	}
	bool move_left{
		if is_slippy {
			return move_slippy([my_cell.cell_left,my_cell.cell_up, my_cell.cell_down], ["L","U","D"]);
		} 
		return move(my_cell.cell_left, "L");
	}
	

	aspect default {
		switch status {
			match "D" {
				draw image_file("../images/elf_down.png") size: 120 /map_size  at: location;
			}
			match "U" {
				draw image_file("../images/elf_up.png") size: 120 /map_size  at: location;

			}
			match "R" {
				draw image_file("../images/elf_right.png") size: 120 /map_size  at: location;

			}
			match "L" {
				draw image_file("../images/elf_left.png") size: 120 /map_size  at: location;

			}
		}
	}
}


grid cell width: map_size height: map_size neighbors: 4 {
	string state <- "F" among:["S", "G","F", "H"];
	//- "S" for Start tile
    //- "G" for Goal tile
    //- "F" for frozen tile
    //- "H" for a tile with a hole
    
    cell cell_down;
    cell cell_up;
    cell cell_right;
    cell cell_left;
    
   
    init {
    	cell_down <- neighbors first_with (each.grid_y > grid_y);
    	cell_up <- neighbors first_with (each.grid_y < grid_y);
    	
    	cell_right <- neighbors first_with (each.grid_x > grid_x);
    	cell_left <- neighbors first_with (each.grid_x < grid_x);
    	
    }
    
	aspect default {
		switch state {
			match "S" {
				draw shape texture:("../images/ice.png") border: #black rotate:-90; 
			}
			match "F" {
				draw shape texture:("../images/ice.png") border: #black rotate:-90; 
			}
			match "H" {
				draw shape texture:("../images/hole.png") border: #black rotate:-90; 
			}
			match "G" {
				draw shape texture:("../images/ice.png") border: #black rotate:-90; 
				draw image_file("../images/goal.png") size: 120 /map_size  at: location;
			}
		}
	}
	
	aspect info {
		switch grid_value {
			match 1.0 {
				draw string("v") color: #black;
			}
			match 2.0 {
				draw string("^") color: #black;
			}
			match 3.0 {
				draw string(">") color: #black;
			}
			match 4.0 {
				draw string("<") color: #black;
			} 
		}
	}
}

grid elf_path width: map_size height: map_size neighbors: 4 frequency: grid_frequency{
	int value <- 0 max: 255 update: (value <= evaporation_per_cycle) ? 0 : value - evaporation_per_cycle;
	rgb color <- rgb(255, 255, 255) update: rgb(255-value, 255, 255-value);
}


experiment Frozenlake type: gui {
	font text <- font("Arial", 14, #bold);
	font title <- font("Arial", 18, #bold);
	
	parameter "Map size" var: map_size <- 8 min:2 max:24 category: "Environnement variables";
	parameter "Is slippery" var: is_slippy category: "Environnement variables";
	parameter "Holes frequencie" var: proba_holes category: "Environnement variables";
	parameter "Distance heuristic" var: dist_heuristic category: "Environnement variables";
	
	parameter "Learning rate" var: learning_rate category: "Q-Learning variables";
	parameter "Gamma" var: gamma category: "Q-Learning variables";
	parameter "Epsilon" var: epsilon category: "Q-Learning variables";
	
	output{
		monitor "Distance to goal" value: elf[0].dist_to_goal;
		monitor "Total rewards" value: elf[0].total_rewards;
		
		display map type: 2d axes: false{
			overlay position: { 50#px,20#px} size: { 1 #px, 1 #px } background: #black border: #white rounded: false 
            	{
                
                draw "Legend" at: {0, 0} anchor: #top_left  color: #black font: title;
                float y <- 50#px;
                
                draw square(40#px) at: {20#px, y} color: #green border: #black;
             	draw "Recent path" at: {60#px, y} anchor: #left_center color: #black font: text;
                
                y <- y + 40#px;
                draw line([{0#px, y}, {40#px, y}])+ 0.5 color: #red;
                draw "Best path" at: {60#px, y} anchor: #left_center color: #black font: text;
            	}
			
			species cell;
			agents "Path" transparency: 0.4 value: elf_path where (each.value > 0);
			graphics "best path" {
				if (best_path_geom != nil) {
					draw best_path_geom + 0.5 color: #red;
				}
			}
			species elf;
		}
//		display q_table type: 2d{
//			overlay position: {10#px, 20#px} size: {1#px, 1#px} background: #white border: #white rounded: false{
//				draw "Legend" at: {0, 0} anchor: #top_left color: #black font: title;
//				float y <- 50#px;
//				
//                draw square(40#px) at: {20#px, y} color: #blue border: #black;
//             	draw "Positive" at: {50#px, y - 10#px} anchor: #left_center color: #black font: text;
//             	draw "reward" at: {50#px, y + 10#px} anchor: #left_center color: #black font: text;
//             	
//             	y <- y + 60#px;
//                draw square(40#px) at: {20#px, y} color: #red border: #black;
//             	draw "Negative" at: {50#px, y - 10#px} anchor: #left_center color: #black font: text;
//             	draw "reward" at: {50#px, y + 10#px} anchor: #left_center color: #black font: text;
//             	
//			}
//			
//			grid cell border: #black;
//			species cell aspect: info;
//			
//		}
		display distribution type: 2d{
			chart "Actions distribution" type: histogram {
				datalist elf[0].all_actions.keys value: elf[0].all_actions.values;
			}
		}
	}
}

[AlexisDrogoul]

I have been able to obtain the NPE with this super simplified model.

model FrozenlakeQL

global {
	map<cell, map<string, float>> q;
	
	init {
		loop c over: cell {
			q[c] <- ["D":: 0.0,"U"::0.0];
		}
	}

	init {
		path the_path <- path_between((cell where (true)), first(cell), last(cell));
	}
	
	reflex {
		string act <- (q[first(cell)].keys) with_min_of (q[first(cell)][each]);
	}
}

grid cell;

experiment Frozenlake type: gui {
	output  {
		display map type: 2d {
			agents "Path" value: cell where (each.grid_value > 0.4);
		}
	}
}

If I change anything in it (i.e. removing the computation of the path, for instance, or the display of the agents) it works. Otherwise it throws the error. Go figure !

I must admit I'm puzzled.