feat(web): Start of Sentence support - part 1 🕊

common/core/desktop/src/kmx/kmx_file.h

@@ -110,6 +110,8 @@ namespace kmx {
 
 #define TSS_COMPARISON 30
 
+#define TSS__KEYMAN_80_MAX  30
+
 /* Keyman 9.0 system stores */
 
 #define TSS_PLATFORM    31
@@ -126,7 +128,23 @@ namespace kmx {
 
 #define TSS_TARGETS         38
 
-#define TSS__MAX        38
+#define TSS__KEYMAN_90_MAX  38
+
+/* Keyman 14.0 system stores */
+
+#define TSS_CASEDKEYS       39
+
+#define TSS__KEYMAN_140_MAX  39
+
+/* Keyman 15.0 system stores */
+
+#define TSS_BEGIN_NEWCONTEXT    40
+#define TSS_BEGIN_POSTKEYSTROKE 41
+#define TSS_LAYERCHANGED     42
+
+#define TSS__KEYMAN_150_MAX  42
+
+#define TSS__MAX        42
 
 /* wm_keyman_control_internal message control codes */
 

common/core/web/input-processor/src/text/inputProcessor.ts

@@ -10,14 +10,16 @@ namespace com.keyman.text {
       baseLayout: 'us'
     }
 
+    private device: utils.DeviceSpec;
     private kbdProcessor: KeyboardProcessor;
     private lngProcessor: prediction.LanguageProcessor;
 
-    constructor(options?: ProcessorInitOptions) {
+    constructor(device: utils.DeviceSpec, options?: ProcessorInitOptions) {
       if(!options) {
         options = InputProcessor.DEFAULT_OPTIONS;
       }
 
+      this.device = device;
       this.kbdProcessor = new KeyboardProcessor(options);
       this.lngProcessor = new prediction.LanguageProcessor();
     }
@@ -50,11 +52,28 @@ namespace com.keyman.text {
       return this.languageProcessor.activeModel;
     }
 
-        /**
+    /**
+     * Tell the currently active keyboard that a new context has been selected,
+     * e.g. by focus change, selection change, keyboard change, etc.
+     *
+     * @param    {Object}   outputTarget  The OutputTarget that has focus
+     * @returns  {Object}                 A RuleBehavior object describing the cumulative effects of
+     *                                    all matched keyboard rules
+     */
+     processNewContextEvent(outputTarget: OutputTarget): RuleBehavior {
+      const ruleBehavior = this.keyboardProcessor.processNewContextEvent(this.device, outputTarget);
+
+      if(ruleBehavior) {
+        ruleBehavior.finalize(this.keyboardProcessor, outputTarget);
+      }
+      return ruleBehavior;
+    }
+
+    /**
      * Simulate a keystroke according to the touched keyboard button element
      *
      * Handles default output and keyboard processing for both OSK and physical keystrokes.
-     * 
+     *
      * @param       {Object}      keyEvent      The abstracted KeyEvent to use for keystroke processing
      * @param       {Object}      outputTarget  The OutputTarget receiving the KeyEvent
      * @returns     {Object}                    A RuleBehavior object describing the cumulative effects of
@@ -105,6 +124,8 @@ namespace com.keyman.text {
       // Current, long-existing assumption - it's DOM-backed.
       let preInputMock = Mock.from(outputTarget);
 
+      const startingLayerId = this.keyboardProcessor.layerId;
+
       // We presently need the true keystroke to run on the FULL context.  That index is still
       // needed for some indexing operations when comparing two different output targets.
       let ruleBehavior = this.keyboardProcessor.processKeystroke(keyEvent, outputTarget);
@@ -113,7 +134,7 @@ namespace com.keyman.text {
       if(keyEvent.kNextLayer) {
         this.keyboardProcessor.selectLayer(keyEvent);
       }
-      
+
       // Should we swallow any further processing of keystroke events for this keydown-keypress sequence?
       if(ruleBehavior != null) {
         let alternates: Alternate[];
@@ -123,7 +144,7 @@ namespace com.keyman.text {
         if(this.languageProcessor.isActive && !ruleBehavior.triggersDefaultCommand) {
           let keyDistribution = keyEvent.keyDistribution;
 
-          // We don't need to track absolute indexing during alternate-generation; 
+          // We don't need to track absolute indexing during alternate-generation;
           // only position-relative, so it's better to use a sliding window for context
           // when making alternates.  (Slightly worse for short text, matters greatly
           // for long text.)
@@ -141,7 +162,7 @@ namespace com.keyman.text {
             let _globalThis = com.keyman.utils.getGlobalObject();
             let timer: () => number;
 
-            // Available by default on `window` in browsers, but _not_ on `global` in Node, 
+            // Available by default on `window` in browsers, but _not_ on `global` in Node,
             // surprisingly.  Since we can't use code dependent on `require` statements
             // at present, we have to condition upon it actually existing.
             if(_globalThis['performance'] && _globalThis['performance']['now']) {
@@ -152,15 +173,15 @@ namespace com.keyman.text {
               TIMEOUT_THRESHOLD = timer() + 16; // + 16ms.
             } // else {
               // We _could_ just use Date.now() as a backup... but that (probably) only matters
-              // when unit testing.  So... we actually don't _need_ time thresholding when in 
+              // when unit testing.  So... we actually don't _need_ time thresholding when in
               // a Node environment.
             // }
 
             // Tracks a minimum probability for keystroke probability.  Anything less will not be
-            // included in alternate calculations. 
+            // included in alternate calculations.
             //
             // Seek to match SearchSpace.EDIT_DISTANCE_COST_SCALE from the predictive-text engine.
-            // Reasoning for the selected value may be seen there.  Short version - keystrokes 
+            // Reasoning for the selected value may be seen there.  Short version - keystrokes
             // that _appear_ very precise may otherwise not even consider directly-neighboring keys.
             let KEYSTROKE_EPSILON = Math.exp(-5);
 
@@ -169,7 +190,7 @@ namespace com.keyman.text {
 
             let activeLayout = this.activeKeyboard.layout(keyEvent.device.formFactor);
             alternates = [];
-    
+
             let totalMass = 0; // Tracks sum of non-error probabilities.
             for(let pair of keyDistribution) {
               if(pair.p < KEYSTROKE_EPSILON) {
@@ -184,7 +205,7 @@ namespace com.keyman.text {
               }
 
               let mock = Mock.from(windowedMock);
-              
+
               let altKey = activeLayout.getLayer(keyEvent.kbdLayer).getKey(pair.keyId);
               if(!altKey) {
                 console.warn("Potential fat-finger key could not be found in layer!");
@@ -193,14 +214,14 @@ namespace com.keyman.text {
 
               let altEvent = altKey.constructKeyEvent(this.keyboardProcessor, keyEvent.device);
               let alternateBehavior = this.keyboardProcessor.processKeystroke(altEvent, mock);
-              
+
               // If alternateBehavior.beep == true, ignore it.  It's a disallowed key sequence,
               // so we expect users to never intend their use.
               //
               // Also possible that this set of conditions fail for all evaluated alternates.
               if(alternateBehavior && !alternateBehavior.beep && pair.p > 0) {
                 let transform: Transform = alternateBehavior.transcription.transform;
-                
+
                 // Ensure that the alternate's token id matches that of the current keystroke, as we only
                 // record the matched rule's context (since they match)
                 transform.id = ruleBehavior.transcription.token;
@@ -224,7 +245,7 @@ namespace com.keyman.text {
         ruleBehavior.finalize(this.keyboardProcessor, outputTarget);
 
         // -- All keystroke (and 'alternate') processing is now complete.  Time to finalize everything! --
-        
+
         // Notify the ModelManager of new input - it's predictive text time!
         if(alternates && alternates.length > 0) {
           ruleBehavior.transcription.alternates = alternates;
@@ -237,6 +258,20 @@ namespace com.keyman.text {
           // For DOM-aware targets, this will trigger a DOM event page designers may listen for.
           outputTarget.doInputEvent();
         }
+
+        // The keyboard may want to take an action after all other keystroke processing is
+        // finished, for example to switch layers. This action may not have any output
+        // but may change system store or variable store values. Given this, we don't need to
+        // save anything about the post behavior, after finalizing it
+
+        // We need to tell the keyboard if the layer has been changed, either by a keyboard rule itself,
+        // or by the touch layout 'nextlayer' control.
+        this.keyboardProcessor.layerChangedStore.set(startingLayerId == this.keyboardProcessor.layerId ? '0' : '1');
+
+        let postRuleBehavior = this.keyboardProcessor.processPostKeystroke(keyEvent.device, outputTarget);
+        if(postRuleBehavior) {
+          postRuleBehavior.finalize(this.keyboardProcessor, outputTarget);
+        }
       }
 
       return ruleBehavior;
@@ -245,6 +280,12 @@ namespace com.keyman.text {
     public resetContext(outputTarget?: OutputTarget) {
       this.keyboardProcessor.resetContext();
       this.languageProcessor.invalidateContext(outputTarget);
+
+      // Let the keyboard do its initial group processing
+      //console.log('processNewContextEvent called from resetContext');
+      if(outputTarget) {
+        this.processNewContextEvent(outputTarget);
+      }
     }
   }
 }

common/core/web/keyboard-processor/src/keyboards/keyboard.ts

@@ -58,12 +58,26 @@ namespace com.keyman.keyboards {
     }
 
     /**
-     * Calls the keyboard's `gs` function, which represents the keyboard source's group(main).
+     * Calls the keyboard's `gs` function, which represents the keyboard source's begin Unicode group.
      */
     process(outputTarget: text.OutputTarget, keystroke: text.KeyEvent): boolean {
       return this.scriptObject['gs'](outputTarget, keystroke);
     }
 
+    /**
+     * Calls the keyboard's `gn` function, which represents the keyboard source's begin newContext group.
+     */
+    processNewContextEvent(outputTarget: text.OutputTarget, keystroke: text.KeyEvent): boolean {
+      return this.scriptObject['gn'] ? this.scriptObject['gn'](outputTarget, keystroke) : false;
+    }
+
+    /**
+     * Calls the keyboard's `gpk` function, which represents the keyboard source's begin postKeystroke group.
+     */
+    processPostKeystroke(outputTarget: text.OutputTarget, keystroke: text.KeyEvent): boolean {
+      return this.scriptObject['gpk'] ? this.scriptObject['gpk'](outputTarget, keystroke) : false;
+    }
+
     get isHollow(): boolean {
       return this.scriptObject == Keyboard.DEFAULT_SCRIPT_OBJECT;
     }

common/core/web/keyboard-processor/src/text/kbdInterface.ts

@@ -185,6 +185,7 @@ namespace com.keyman.text {
 
     static readonly TSS_LAYER:    number = 33;
     static readonly TSS_PLATFORM: number = 31;
+    static readonly TSS_LAYERCHANGED: number = 42;
 
     systemStores: {[storeID: number]: SystemStore};
 
@@ -201,6 +202,7 @@ namespace com.keyman.text {
 
       this.systemStores[KeyboardInterface.TSS_PLATFORM] = new PlatformSystemStore(this);
       this.systemStores[KeyboardInterface.TSS_LAYER] = new MutableSystemStore(KeyboardInterface.TSS_LAYER, 'default');
+      this.systemStores[KeyboardInterface.TSS_LAYERCHANGED] = new MutableSystemStore(KeyboardInterface.TSS_LAYERCHANGED, '0');
 
       this.variableStoreSerializer = variableStoreSerializer;
     }
@@ -948,20 +950,59 @@ namespace com.keyman.text {
       this.output(1, outputTarget, "");
     }
 
+    /**
+     * Function     processNewContextEvent
+     * Scope        Private
+     * @param       {Object}        outputTarget   The target receiving input
+     * @param       {Object}        keystroke      The input keystroke (with its properties) to be mapped by the keyboard.
+     * Description  Calls the keyboard's `begin newContext` group
+     * @returns     {RuleBehavior}  Record of commands and state changes that result from executing `begin NewContext`
+     */
+    processNewContextEvent(outputTarget: OutputTarget, keystroke: KeyEvent): RuleBehavior {
+      if(!this.activeKeyboard) {
+        throw "No active keyboard for keystroke processing!";
+      }
+      return this.process(this.activeKeyboard.processNewContextEvent.bind(this.activeKeyboard), outputTarget, keystroke);
+    }
+
+    /**
+     * Function     processPostKeystroke
+     * Scope        Private
+     * @param       {Object}        outputTarget   The target receiving input
+     * @param       {Object}        keystroke      The input keystroke with relevant properties to be mapped by the keyboard.
+     * Description  Calls the keyboard's `begin postKeystroke` group
+     * @returns     {RuleBehavior}  Record of commands and state changes that result from executing `begin PostKeystroke`
+     */
+    processPostKeystroke(outputTarget: OutputTarget, keystroke: KeyEvent): RuleBehavior {
+      if(!this.activeKeyboard) {
+        throw "No active keyboard for keystroke processing!";
+      }
+      return this.process(this.activeKeyboard.processPostKeystroke.bind(this.activeKeyboard), outputTarget, keystroke);
+    }
+
     /**
      * Function     processKeystroke
      * Scope        Private
-     * @param       {Object}        element     The page element receiving input
+     * @param       {Object}        outputTarget   The target receiving input
      * @param       {Object}        keystroke   The input keystroke (with its properties) to be mapped by the keyboard.
      * Description  Encapsulates calls to keyboard input processing.
-     * @returns     {number}        0 if no match is made, otherwise 1.
+     * @returns     {RuleBehavior}  Record of commands and state changes that result from executing `begin Unicode`
      */
     processKeystroke(outputTarget: OutputTarget, keystroke: KeyEvent): RuleBehavior {
+      if(!this.activeKeyboard) {
+        throw "No active keyboard for keystroke processing!";
+      }
+      return this.process(this.activeKeyboard.process.bind(this.activeKeyboard), outputTarget, keystroke);
+    }
+
+    private process(callee, outputTarget: OutputTarget, keystroke: KeyEvent): RuleBehavior {
       // Clear internal state tracking data from prior keystrokes.
       if(!outputTarget) {
         throw "No target specified for keyboard output!";
       } else if(!this.activeKeyboard) {
         throw "No active keyboard for keystroke processing!";
+      } else if(!callee) {
+        throw "No callee for keystroke processing!";
       }
 
       outputTarget.invalidateSelection();
@@ -984,7 +1025,7 @@ namespace com.keyman.text {
 
       // Calls the start-group of the active keyboard.
       this.activeTargetOutput = outputTarget;
-      var matched = this.activeKeyboard.process(outputTarget, keystroke);
+      var matched = callee(outputTarget, keystroke);
       this.activeTargetOutput = null;
 
       // Finalize the rule's results.

common/core/web/keyboard-processor/src/text/keyboardProcessor.ts

@@ -91,6 +91,10 @@ namespace com.keyman.text {
       return this.keyboardInterface.systemStores[KeyboardInterface.TSS_LAYER] as MutableSystemStore;
     }
 
+    public get layerChangedStore(): MutableSystemStore {
+      return this.keyboardInterface.systemStores[KeyboardInterface.TSS_LAYERCHANGED] as MutableSystemStore;
+    }
+
     public get layerId(): string {
       return this.layerStore.value;
     }
@@ -207,6 +211,27 @@ namespace com.keyman.text {
       }
     }
 
+    constructNullKeyEvent(device: utils.DeviceSpec): KeyEvent {
+      const keyEvent = new KeyEvent();
+      keyEvent.Lcode = 0;
+      keyEvent.kName = '';
+      keyEvent.device = device;
+      this.setSyntheticEventDefaults(keyEvent);
+      return keyEvent;
+    }
+
+    processNewContextEvent(device: utils.DeviceSpec, outputTarget: OutputTarget): RuleBehavior {
+      return this.activeKeyboard ?
+        this.keyboardInterface.processNewContextEvent(outputTarget, this.constructNullKeyEvent(device)) :
+        null;
+    }
+
+    processPostKeystroke(device: utils.DeviceSpec, outputTarget: OutputTarget): RuleBehavior {
+      return this.activeKeyboard ?
+        this.keyboardInterface.processPostKeystroke(outputTarget, this.constructNullKeyEvent(device)) :
+        null;
+    }
+
     processKeystroke(keyEvent: KeyEvent, outputTarget: OutputTarget): RuleBehavior {
       var matchBehavior: RuleBehavior;
 

common/test/keyboards/start_of_sentence_3621/.gitignore

@@ -0,0 +1 @@
+build/

common/test/keyboards/start_of_sentence_3621/HISTORY.md

@@ -0,0 +1,6 @@
+start_of_sentence_3621 Change History
+====================
+
+1.0 (2021-11-22)
+----------------
+* Created by