New keycode macro (XP) for shifted character pairs using UNICODEMAP + bug fixes and improvements (#4803)

* Expose unicode_saved_mods

* Add UNICODEMAP shift pair functionality and XS keycode

* Add XS to keycode reference documentation

* Pick pair index based on both Shift and Caps Lock state

* Add XS to Unicode feature docs

* Clean up process_unicode* headers

* Extract unicode_map index calculation into function

* Pick pair index as XOR rather than OR of Shift and Caps states

* unicode_input_start() has to be called before the unicode_map index is calculated

* Replace unicodemap_input_error() with more generic unicode_input_cancel()

* Replace register+tap+unregister with tap_code16(LCTL(LSFT(KC_U)))

* UNICODE_OSX_KEY → UNICODE_KEY_OSX, UNICODE_WINC_KEY → UNICODE_KEY_WINC

* Make keycode range checks more robust

* Fix keycode range checks for different input modes

* Add UNICODE_KEY_LNX, update docs

* QK_UNICODEMAP_SHIFT → QK_UNICODEMAP_PAIR

* XS → XP, update docs

* Tweak Unicode docs

* Use recently added MOD_MASK_SHIFT and IS_HOST_LED_ON helpers

* Update Unicode table in docs/keycodes.md

* Update Unicode docs per review comments

* Replace references to Mac OS X with macOS in Unicode docs

* As of v0.9.0, WinCompose supports all possible code points

* Expand descriptions in XP docs

* Update keycode table and cycling docs

* Further expand cycling docs

1 files changed, 23 insertions, 13 deletions

diff --git a/quantum/process_keycode/process_unicodemap.c b/quantum/process_keycode/process_unicodemap.c
index 3274027613..b887879860 100644
--- a/quantum/process_keycode/process_unicodemap.c
+++ b/quantum/process_keycode/process_unicodemap.c
@@ -15,7 +15,6 @@
  */
 
 #include "process_unicodemap.h"
-#include "process_unicode_common.h"
 
 void register_hex32(uint32_t hex) {
   bool onzerostart = true;
@@ -38,28 +37,39 @@ void register_hex32(uint32_t hex) {
 }
 
 __attribute__((weak))
-void unicodemap_input_error() {}
+uint16_t unicodemap_index(uint16_t keycode) {
+  if (keycode >= QK_UNICODEMAP_PAIR) {
+    // Keycode is a pair: extract index based on Shift / Caps Lock state
+    uint16_t index = keycode - QK_UNICODEMAP_PAIR;
+
+    bool shift = unicode_saved_mods & MOD_MASK_SHIFT, caps = IS_HOST_LED_ON(USB_LED_CAPS_LOCK);
+    if (shift ^ caps) { index >>= 7; }
+
+    return index & 0x7F;
+  } else {
+    // Keycode is a regular index
+    return keycode - QK_UNICODEMAP;
+  }
+}
 
 bool process_unicodemap(uint16_t keycode, keyrecord_t *record) {
-  if ((keycode & QK_UNICODEMAP) == QK_UNICODEMAP && record->event.pressed) {
-    uint16_t index = keycode - QK_UNICODEMAP;
-    uint32_t code = pgm_read_dword(unicode_map + index);
+  if (keycode >= QK_UNICODEMAP && keycode <= QK_UNICODEMAP_PAIR_MAX && record->event.pressed) {
+    unicode_input_start();
+
+    uint32_t code = pgm_read_dword(unicode_map + unicodemap_index(keycode));
     uint8_t input_mode = get_unicode_input_mode();
 
-    if (code > 0xFFFF && code <= 0x10FFFF && input_mode == UC_OSX) {
-      // Convert to UTF-16 surrogate pair
+    if (code > 0x10FFFF || (code > 0xFFFF && input_mode == UC_WIN)) {
+      // Character is out of range supported by the platform
+      unicode_input_cancel();
+    } else if (code > 0xFFFF && input_mode == UC_OSX) {
+      // Convert to UTF-16 surrogate pair on Mac
       code -= 0x10000;
       uint32_t lo = code & 0x3FF, hi = (code & 0xFFC00) >> 10;
-
-      unicode_input_start();
       register_hex32(hi + 0xD800);
       register_hex32(lo + 0xDC00);
       unicode_input_finish();
-    } else if ((code > 0x10FFFF && input_mode == UC_OSX) || (code > 0xFFFFF && input_mode == UC_LNX)) {
-      // Character is out of range supported by the OS
-      unicodemap_input_error();
     } else {
-      unicode_input_start();
       register_hex32(code);
       unicode_input_finish();
     }