Merge branch 'dev' into mpf_service

appveyor.yml

@@ -17,7 +17,8 @@ environment:
 
 install:
   - "git submodule update --init --recursive"
-  - "%PYTHON%\\python.exe -m pip install -U setuptools wheel pip mock twine pypiwin32==219"
+  - "%PYTHON%\\python.exe -m pip install -U setuptools wheel pip mock twine"
+  - "%PYTHON%\\python.exe -m pip install -e ."
 
 build: off
 

mpf/_version.py

@@ -10,7 +10,7 @@
 
 """
 
-__version__ = '0.50.0-dev.52'
+__version__ = '0.50.0-dev.55'
 '''The full version of MPF.'''
 
 __short_version__ = '0.50'

mpf/core/data_manager.py

@@ -110,31 +110,6 @@ def save_all(self, data):
         self.data = data
         self._trigger_save()
 
-    def save_key(self, key, value):
-        """Update an individual key and then write the entire dictionary to disk.
-
-        Args:
-            key: String name of the key to add/update.
-            value: Value of the key
-        """
-        try:
-            self.data[key] = value
-        except TypeError:
-            self.debug_log.warning('In-memory copy of %s is invalid. Re-creating', self.filename)
-            # todo should we reload from disk here?
-            self.data = dict()
-            self.data[key] = value
-
-        self._trigger_save()
-
-    def remove_key(self, key):
-        """Remove key by name."""
-        try:
-            del self.data[key]
-            self._trigger_save()
-        except KeyError:
-            pass
-
     def _writing_thread(self):  # pragma: no cover
         # prevent early writes at start-up
         time.sleep(self.min_wait_secs)

mpf/core/machine.py

@@ -352,7 +352,7 @@ def _load_machine_vars(self) -> None:
             if ('expire' in settings and settings['expire'] and
                     settings['expire'] < current_time):
 
-                settings['value'] = 0
+                continue
 
             self.set_machine_var(name=name, value=settings['value'])
 
@@ -378,9 +378,9 @@ def _load_initial_machine_vars(self) -> None:
         for name, element in config.items():
             if name not in self.machine_vars:
                 element = self.config_validator.validate_config("machine_vars", copy.deepcopy(element))
-                self.configure_machine_var(name=name, persist=element['persist'])
                 self.set_machine_var(name=name,
                                      value=Util.convert_to_type(element['initial_value'], element['value_type']))
+            self.configure_machine_var(name=name, persist=element.get('persist', False))
 
     def _set_machine_path(self) -> None:
         """Add the machine folder to sys.path so we can import modules from it."""
@@ -777,13 +777,13 @@ def _platform_stop(self) -> None:
     def _write_machine_var_to_disk(self, name: str) -> None:
         """Write value to disk."""
         if self.machine_vars[name]['persist'] and self.config['mpf']['save_machine_vars_to_disk']:
-            disk_var = CaseInsensitiveDict()
-            disk_var['value'] = self.machine_vars[name]['value']
-
-            if self.machine_vars[name]['expire_secs']:
-                disk_var['expire'] = self.clock.get_time() + self.machine_vars[name]['expire_secs']
+            self._write_machine_vars_to_disk()
 
-            self.machine_var_data_manager.save_key(name, disk_var)
+    def _write_machine_vars_to_disk(self):
+        """Update machine vars on disk."""
+        self.machine_var_data_manager.save_all(
+            {name: {"value": var["value"], "expire": var['expire_secs']}
+             for name, var in self.machine_vars.items() if var["persist"]})
 
     def get_machine_var(self, name: str) -> Any:
         """Return the value of a machine variable.
@@ -814,21 +814,16 @@ def configure_machine_var(self, name: str, persist: bool, expire_secs: int = Non
                 disk so it's available the next time MPF boots.
             expire_secs: Optional number of seconds you'd like this variable
                 to persist on disk for. When MPF boots, if the expiration time
-                of the variable is in the past, it will be loaded with a value
-                of 0. For example, this lets you write the number of credits on
+                of the variable is in the past, it will not be loaded.
+                For example, this lets you write the number of credits on
                 the machine to disk to persist even during power off, but you
                 could set it so that those only stay persisted for an hour.
         """
         if name not in self.machine_vars:
-            var = CaseInsensitiveDict()
-
-            var['value'] = None
-            var['persist'] = persist
-            var['expire_secs'] = expire_secs
-            self.machine_vars[name] = var
+            self.machine_vars[name] = {'value': None, 'persist': persist, 'expire_secs': expire_secs}
         else:
             self.machine_vars[name]['persist'] = persist
-            self.machine_vars[name]['expire_sec'] = expire_secs
+            self.machine_vars[name]['expire_secs'] = expire_secs
 
     def set_machine_var(self, name: str, value: Any) -> None:
         """Set the value of a machine variable.
@@ -895,7 +890,7 @@ def remove_machine_var(self, name: str) -> None:
         """
         try:
             del self.machine_vars[name]
-            self.machine_var_data_manager.remove_key(name)
+            self._write_machine_vars_to_disk()
         except KeyError:
             pass
 
@@ -912,7 +907,8 @@ def remove_machine_var_search(self, startswith: str = '', endswith: str = '') ->
         for var in list(self.machine_vars.keys()):
             if var.startswith(startswith) and var.endswith(endswith):
                 del self.machine_vars[var]
-                self.machine_var_data_manager.remove_key(var)
+
+        self._write_machine_vars_to_disk()
 
     def get_platform_sections(self, platform_section: str, overwrite: str) -> "SmartVirtualHardwarePlatform":
         """Return platform section."""

mpf/core/mode_controller.py

@@ -192,7 +192,10 @@ def _load_mode_from_machine_folder(self, mode_string: str, code_path: str) -> Op
                 self.machine.config['mpf']['paths']['modes'] + '.' +
                 self._machine_mode_folders[mode_string] + '.code.' +
                 file_name)
-        except ImportError:
+        except ImportError as e:
+            # do not hide import error in mode
+            if e.name != file_name:
+                raise e
             return None
 
         return getattr(i, class_name, None)
@@ -205,7 +208,10 @@ def _load_mode_from_full_path(code_path: str) -> Optional[Callable[..., Mode]]:
         """
         try:
             return Util.string_to_class(code_path)
-        except ImportError:
+        except ImportError as e:
+            # do not hide import error in mode
+            if e.name != code_path.split('.')[-1]:
+                raise e
             return None
 
     def _load_mode_code(self, mode_string: str, code_path: str) -> Callable[..., Mode]:

mpf/devices/multiball_lock.py

@@ -168,6 +168,7 @@ def _register_handlers(self):
     def _unregister_handlers(self):
         # unregister ball_enter handlers
         self.machine.events.remove_handler(self._lock_ball)
+        self.machine.events.remove_handler(self._post_events)
 
     @property
     def is_virtually_full(self):
@@ -284,6 +285,7 @@ def _post_events(self, device, **kwargs):
         del kwargs
         for event in self._events[device]:
             self.machine.events.post(**event)
+        self._events[device] = []
 
     def _request_new_balls(self, balls):
         """Request new ball to playfield."""

mpf/devices/shot_group.py

@@ -110,9 +110,12 @@ def _hit(self, advancing, **kwargs):
         """One of the member shots in this shot group was hit.
 
         Args:
-            kwarg: unused
+            kwarg: {
+                profile: the current profile of the member shot that was hit
+                state: the current state of the member shot that was hit
+                advancing: boolean of whether the state is advancing
+            }
         """
-        del kwargs
         if advancing:
             self._check_for_complete()
 
@@ -121,6 +124,11 @@ def _hit(self, advancing, **kwargs):
         desc: A member shots in the shot group called (shot_group)
         has been hit.
         '''
+        self.machine.events.post("{}_{}_hit".format(self.name, kwargs['state']))
+        '''event: (shot_group)_(state)_hit
+        desc: A member shot with state (state) in the shot group (shot_group)
+        has been hit.
+        '''
 
     @event_handler(9)
     def enable_rotation(self, **kwargs):

mpf/platforms/opp/opp.py

@@ -65,7 +65,7 @@ def __init__(self, machine) -> None:
         # TODO: refactor this into the OPPNeopixelCard
         self.neoDict = dict()               # type: Dict[str, OPPNeopixel]
         self.numGen2Brd = 0
-        self.gen2AddrArr = {}               # type: Dict[str, List[int]]
+        self.gen2AddrArr = {}               # type: Dict[str, Dict[int, int]]
         self.badCRC = 0
         self.minVersion = 0xffffffff
         self._poll_task = {}                # type: Dict[str, asyncio.Task]
@@ -156,6 +156,19 @@ def process_received_message(self, chain_serial, msg):
             #  until they come back
             self.opp_connection[chain_serial].lost_synch()
 
+    @staticmethod
+    def _get_numbers(mask):
+        number = 0
+        ref = 1
+        result = []
+        while mask > ref:
+            if mask & ref:
+                result.append(number)
+            number += 1
+            ref = ref << 1
+
+        return result
+
     def get_info_string(self):
         """Dump infos about boards."""
         if not self.serial_connections:
@@ -164,22 +177,33 @@ def get_info_string(self):
         infos = "Connected CPUs:\n"
         for connection in self.serial_connections:
             infos += " - Port: {} at {} baud\n".format(connection.port, connection.baud)
+            for board_id, board_firmware in self.gen2AddrArr[connection.chain_serial].items():
+                if board_firmware is None:
+                    infos += " -> Board: 0x{:02x} Firmware: broken\n".format(board_id)
+                else:
+                    infos += " -> Board: 0x{:02x} Firmware: 0x{:02x}\n".format(board_id, board_firmware)
 
         infos += "\nIncand cards:\n"
         for incand in self.opp_incands:
-            infos += " - CPU: {} Card: {} Mask: {}".format(incand.chain_serial, incand.cardNum, incand.mask)
+            infos += " - CPU: {} Board: 0x{:02x} Card: {} Numbers: {}\n".format(incand.chain_serial, incand.addr,
+                                                                                incand.cardNum,
+                                                                                self._get_numbers(incand.mask))
 
         infos += "\nInput cards:\n"
         for inputs in self.opp_inputs:
-            infos += " - CPU: {} Card: {} Mask: {}".format(inputs.chain_serial, inputs.cardNum, inputs.mask)
+            infos += " - CPU: {} Board: 0x{:02x} Card: {} Numbers: {}\n".format(inputs.chain_serial, inputs.addr,
+                                                                                inputs.cardNum,
+                                                                                self._get_numbers(inputs.mask))
 
-        infos += "\nInput coils:\n"
+        infos += "\nSolenoid cards:\n"
         for outputs in self.opp_solenoid:
-            infos += " - CPU: {} Card: {} Mask: {}".format(outputs.chain_serial, outputs.cardNum, outputs.mask)
+            infos += " - CPU: {} Board: 0x{:02x} Card: {} Numbers: {}\n".format(outputs.chain_serial, outputs.addr,
+                                                                                outputs.cardNum,
+                                                                                self._get_numbers(outputs.mask))
 
         infos += "\nLEDs:\n"
         for leds in self.opp_neopixels:
-            infos += " - CPU: {} Card: {}".format(leds.chain_serial, leds.cardNum)
+            infos += " - CPU: {} Board: 0x{:02x} Card: {}".format(leds.chain_serial, leds.addr, leds.cardNum)
 
         return infos
 
@@ -284,17 +308,18 @@ def inv_resp(self, chain_serial, msg):
             chain_serial: Serial of the chain which received the message.
             msg: Message to parse.
         """
-        # TODO: use chain_serial/move to serial communicator
-        self.log.debug("Received Inventory Response:%s", "".join(" 0x%02x" % b for b in msg))
+        self.log.debug("Received Inventory Response: %s for %s", "".join(" 0x%02x" % b for b in msg), chain_serial)
 
         index = 1
-        self.gen2AddrArr[chain_serial] = []
+        self.gen2AddrArr[chain_serial] = {}
         while msg[index] != ord(OppRs232Intf.EOM_CMD):
             if (msg[index] & ord(OppRs232Intf.CARD_ID_TYPE_MASK)) == ord(OppRs232Intf.CARD_ID_GEN2_CARD):
                 self.numGen2Brd += 1
-                self.gen2AddrArr[chain_serial].append(msg[index])
+                self.gen2AddrArr[chain_serial][msg[index]] = None
+            else:
+                self.log.warning("Invalid inventory response %s for %s.", msg[index], chain_serial)
             index += 1
-        self.log.debug("Found %d Gen2 OPP boards.", self.numGen2Brd)
+        self.log.debug("Found %d Gen2 OPP boards on %s.", self.numGen2Brd, chain_serial)
 
     @staticmethod
     def eom_resp(chain_serial, msg):
@@ -441,6 +466,12 @@ def vers_resp(self, chain_serial, msg):
             self.log.debug("Firmware version: %d.%d.%d.%d", msg[curr_index + 2],
                            msg[curr_index + 3], msg[curr_index + 4],
                            msg[curr_index + 5])
+            if msg[curr_index] not in self.gen2AddrArr[chain_serial]:
+                self.log.warning("Got firmware response for %s but not in inventory at %s", msg[curr_index],
+                                 chain_serial)
+            else:
+                self.gen2AddrArr[chain_serial][msg[curr_index]] = version
+
             if version < self.minVersion:
                 self.minVersion = version
             if version == BAD_FW_VERSION:

mpf/platforms/opp/opp_switch.py

@@ -40,6 +40,7 @@ def __init__(self, chain_serial, addr, inp_dict, inp_addr_dict):
         self.log = logging.getLogger('OPPMatrixCard')
         self.chain_serial = chain_serial
         self.addr = addr
+        self.mask = 0xFFFFFFFFFFFFFFFF << 32  # create fake mask
         self.isMatrix = True
         self.oldState = [0, 0]
         self.cardNum = str(addr - ord(OppRs232Intf.CARD_ID_GEN2_CARD))

mpf/tests/machine_files/machine_vars/config/config.yaml

@@ -0,0 +1,15 @@
+#config_version=5
+
+machine_vars:
+  test1:
+    initial_value: 4
+    value_type: int
+    persist: True
+  test2:
+    initial_value: '5'
+    value_type: str
+    persist: True
+  test3:
+    initial_value: 6
+    value_type: int
+    persist: False

mpf/tests/test_DataManager.py

@@ -38,7 +38,7 @@ def test_save_and_load(self):
         open_mock = mock_open(read_data="")
         with patch('mpf.file_interfaces.yaml_interface.open', open_mock, create=True):
             with patch('mpf.core.data_manager.os.replace') as move_mock:
-                manager.save_key("hallo", "world")
+                manager.save_all({"hallo": "world"})
                 while not move_mock.called:
                     time.sleep(.00001)
                 open_mock().write.assert_called_once_with('hallo: world\n')

mpf/tests/test_MachineVariables.py

@@ -7,11 +7,20 @@
 
 class TestMachineVariables(MpfTestCase):
 
+    def getConfigFile(self):
+        return 'config.yaml'
+
+    def getMachinePath(self):
+        return 'tests/machine_files/machine_vars/'
+
     def _get_mock_data(self):
         return {"machine_vars": {"player2_score": {"value": 118208660},
                                  "player3_score": {"value": 17789290},
                                  "player4_score": {"value": 3006600},
-                                 "another_score": {"value": 123}},
+                                 "another_score": {"value": 123},
+                                 "expired_value": {"value": 23, "expire": self.clock.get_time() - 100},
+                                 "not_expired_value": {"value": 24, "expire": self.clock.get_time() + 100},
+                                 "test1": {"value": 42}},
                 }
 
     def testSystemInfoVariables(self):
@@ -27,7 +36,18 @@ def testSystemInfoVariables(self):
         self.assertEqual(extended_version, self.machine.get_machine_var("mpf_extended_version"))
 
     def testVarLoadAndRemove(self):
+        self.assertFalse(self.machine.is_machine_var("expired_value"))
+        self.assertTrue(self.machine.is_machine_var("not_expired_value"))
         self.assertTrue(self.machine.is_machine_var("player2_score"))
+        # should always persist
+        #self.assertTrue(self.machine.machine_vars["player2_score"]["persist"])
+        # random variable does not persist
+        self.assertFalse(self.machine.machine_vars["another_score"]["persist"])
+        # configured to persist
+        self.assertTrue(self.machine.machine_vars["test1"]["persist"])
+        self.assertTrue(self.machine.machine_vars["test2"]["persist"])
+        # configured to not persist
+        self.assertFalse(self.machine.machine_vars["test3"]["persist"])
         self.assertEqual(118208660, self.machine.get_machine_var("player2_score"))
 
         self.machine.remove_machine_var("player2_score")
@@ -45,7 +65,8 @@ def testVarLoadAndRemove(self):
         self.advance_time_and_run(10)
 
         self.machine.machine_var_data_manager._trigger_save.assert_called_with()
-        self.assertEqual({'another_score': {'value': 123}}, self.machine.machine_var_data_manager.data)
+        self.assertEqual({'test1': {'value': 42, 'expire': None}, 'test2': {'value': '5', 'expire': None}},
+                         self.machine.machine_var_data_manager.data)
 
 
 class TestMalformedMachineVariables(MpfTestCase):