Terminal: measure characters with Unicode 11 widths, so an emoji takes the two columns the remote counts

docs/manual-checks.md

@@ -322,6 +322,30 @@ closing the only open window ends the process, so the real window is shown while
 Nothing appearing at all means composition is running in the same dispatcher turn that has to paint the
 panel, which is what posting it at `DispatcherPriority.Background` prevents.
 
+### 1.13 An emoji takes two columns, and the line after it stays where the shell thinks it is
+
+Open a shell on a Linux host and run:
+
+```
+printf '😀|\n12|\n日|\n'
+```
+
+**Pass:** the three bars line up in one column. The emoji, the two digits and the CJK character each take two
+columns, which is what the remote counted.
+
+Then type `echo 😀😀 hello` at the prompt without pressing Enter, press **←** five times, and type `X`.
+
+**Pass:** the `X` appears between `he` and `llo`, which is where bash put it, and the line does not redraw over
+itself.
+
+**What the tests already own.** `RendererUnicodeTests` proves the page loads `@xterm/addon-unicode11` and
+selects its table before the pane opens. What is left here is xterm itself drawing to that table.
+
+**Failure means:** the emoji's bar sits one column left of the others, or the `X` lands a column away from the
+cursor. The pane is on xterm's default Unicode 6 widths, where an emoji is one column; check that
+`terminal.html` still loads `vendor/addon-unicode11.js` and that `AvaloniaTerminalAssetProvider` serves it.
+A script the provider does not list is a 404 the page never reports.
+
 ---
 
 ## Phase 2 — Known Hosts as its own page

src/DodoSSH.Client.Shell/Terminal/AvaloniaTerminalAssetProvider.cs

@@ -24,6 +24,7 @@ internal sealed class AvaloniaTerminalAssetProvider : ITerminalAssetProvider
         ("/vendor/xterm.css", "vendor/xterm.css", "text/css; charset=utf-8"),
         ("/vendor/addon-fit.js", "vendor/addon-fit.js", "text/javascript; charset=utf-8"),
         ("/vendor/addon-webgl.js", "vendor/addon-webgl.js", "text/javascript; charset=utf-8"),
+        ("/vendor/addon-unicode11.js", "vendor/addon-unicode11.js", "text/javascript; charset=utf-8"),
     ];
 
     private readonly Dictionary<string, TerminalAsset> assets = new(StringComparer.Ordinal);

src/DodoSSH.Client.Shell/WebAssets/terminal.html

@@ -52,6 +52,7 @@
   <script src="/vendor/xterm.js"></script>
   <script src="/vendor/addon-fit.js"></script>
   <script src="/vendor/addon-webgl.js"></script>
+  <script src="/vendor/addon-unicode11.js"></script>
   <script src="/terminal.js"></script>
 </body>
 </html>

src/DodoSSH.Client.Shell/WebAssets/terminal.js

@@ -387,6 +387,28 @@ function createSession(sessionId) {
 
   const fit = new FitAddon.FitAddon();
   term.loadAddon(fit);
+
+  /*
+    ── CHARACTER WIDTHS FROM UNICODE 11, NOT 6 ────────────────────────────────────────────────────────
+
+    How many columns a character takes is not something the terminal and the remote can each decide for
+    themselves: the shell's line editor counts columns to know where the cursor is, and the terminal
+    counts them to know where to draw. When the two disagree, every character after the one they
+    disagree about lands a column away from where the remote believes it is — a prompt that redraws over
+    itself, a cursor one cell off, a line that wraps early.
+
+    xterm.js ships with Unicode 6's table, which predates emoji as a wide class. So 😀 was one column
+    here and two on any current remote (glibc, musl and every modern wcwidth follow Unicode 9 and later),
+    and a line with an emoji in it drifted from the first emoji on. Measured against xterm.js 6.0
+    headless: 😀 took one cell as shipped and two with this addon, and CJK, combining marks and every
+    escape sequence behaved the same either way.
+
+    Before open() so the first frame is measured with the right table. activeVersion is a proposed API,
+    which is what allowProposedApi above is already on for.
+  */
+  term.loadAddon(new Unicode11Addon.Unicode11Addon());
+  term.unicode.activeVersion = '11';
+
   term.open(pane);
 
   // WebGL where it is available. Falling back rather than failing matters because a software

src/DodoSSH.Client.Shell/WebAssets/vendor/README.md

@@ -12,18 +12,21 @@ All of the following are MIT licensed, the same licence as this repository.
 | `xterm.css` | `@xterm/xterm` | 6.0.0 | `https://unpkg.com/@xterm/xterm@6.0.0/css/xterm.css` |
 | `addon-fit.js` | `@xterm/addon-fit` | 0.11.0 | `https://unpkg.com/@xterm/addon-fit@0.11.0/lib/addon-fit.js` |
 | `addon-webgl.js` | `@xterm/addon-webgl` | 0.19.0 | `https://unpkg.com/@xterm/addon-webgl@0.19.0/lib/addon-webgl.js` |
+| `addon-unicode11.js` | `@xterm/addon-unicode11` | 0.9.0 | `https://unpkg.com/@xterm/addon-unicode11@0.9.0/lib/addon-unicode11.js` |
 
 Copyright (c) 2017 The xterm.js authors. See <https://github.com/xtermjs/xterm.js> for the full
 licence text.
 
-These are the UMD builds, so they attach to `globalThis`: `Terminal`, `FitAddon.FitAddon` and
+These are the UMD builds, so they attach to `globalThis`: `Terminal`, `FitAddon.FitAddon`,
-`WebglAddon.WebglAddon`. `terminal.js` depends on those exact names, so check them after any upgrade
+`WebglAddon.WebglAddon` and `Unicode11Addon.Unicode11Addon`. `terminal.js` depends on those exact names, so check them after any upgrade
 — a bundle that switched to ES modules would load without error and leave `Terminal` undefined.
 
 ## Upgrading
 
-1. Download the four files at the new versions and update the table above.
+1. Download the five files at the new versions and update the table above.
-2. Confirm the UMD global names are unchanged.
+2. Confirm the UMD global names are unchanged, and that `term.unicode.activeVersion = '11'` still selects
+   the addon's table: a version string xterm no longer recognises throws, and an addon that registered
+   under another name would leave the page on Unicode 6 widths without a word.
 3. Confirm the right-click still belongs to `terminal.js`. Two things about xterm are load-bearing for it
    and neither is a global: `rightClickSelectsWord` has to still be an option — the page sets it false, and
    an option xterm no longer knows is silently ignored — and xterm has to still register its own right-click

src/DodoSSH.Client.Shell/WebAssets/vendor/addon-unicode11.js

@@ -0,0 +1,2 @@
+!function(e,t){"object"==typeof exports&&"object"==typeof module?module.exports=t():"function"==typeof define&&define.amd?define([],t):"object"==typeof exports?exports.Unicode11Addon=t():e.Unicode11Addon=t()}(globalThis,(()=>(()=>{"use strict";var e={384:(e,t,s)=>{Object.defineProperty(t,"__esModule",{value:!0}),t.UnicodeV11=void 0;const r=s(765),n=[[768,879],[1155,1161],[1425,1469],[1471,1471],[1473,1474],[1476,1477],[1479,1479],[1536,1541],[1552,1562],[1564,1564],[1611,1631],[1648,1648],[1750,1757],[1759,1764],[1767,1768],[1770,1773],[1807,1807],[1809,1809],[1840,1866],[1958,1968],[2027,2035],[2045,2045],[2070,2073],[2075,2083],[2085,2087],[2089,2093],[2137,2139],[2259,2306],[2362,2362],[2364,2364],[2369,2376],[2381,2381],[2385,2391],[2402,2403],[2433,2433],[2492,2492],[2497,2500],[2509,2509],[2530,2531],[2558,2558],[2561,2562],[2620,2620],[2625,2626],[2631,2632],[2635,2637],[2641,2641],[2672,2673],[2677,2677],[2689,2690],[2748,2748],[2753,2757],[2759,2760],[2765,2765],[2786,2787],[2810,2815],[2817,2817],[2876,2876],[2879,2879],[2881,2884],[2893,2893],[2902,2902],[2914,2915],[2946,2946],[3008,3008],[3021,3021],[3072,3072],[3076,3076],[3134,3136],[3142,3144],[3146,3149],[3157,3158],[3170,3171],[3201,3201],[3260,3260],[3263,3263],[3270,3270],[3276,3277],[3298,3299],[3328,3329],[3387,3388],[3393,3396],[3405,3405],[3426,3427],[3530,3530],[3538,3540],[3542,3542],[3633,3633],[3636,3642],[3655,3662],[3761,3761],[3764,3772],[3784,3789],[3864,3865],[3893,3893],[3895,3895],[3897,3897],[3953,3966],[3968,3972],[3974,3975],[3981,3991],[3993,4028],[4038,4038],[4141,4144],[4146,4151],[4153,4154],[4157,4158],[4184,4185],[4190,4192],[4209,4212],[4226,4226],[4229,4230],[4237,4237],[4253,4253],[4448,4607],[4957,4959],[5906,5908],[5938,5940],[5970,5971],[6002,6003],[6068,6069],[6071,6077],[6086,6086],[6089,6099],[6109,6109],[6155,6158],[6277,6278],[6313,6313],[6432,6434],[6439,6440],[6450,6450],[6457,6459],[6679,6680],[6683,6683],[6742,6742],[6744,6750],[6752,6752],[6754,6754],[6757,6764],[6771,6780],[6783,6783],[6832,6846],[6912,6915],[6964,6964],[6966,6970],[6972,6972],[6978,6978],[7019,7027],[7040,7041],[7074,7077],[7080,7081],[7083,7085],[7142,7142],[7144,7145],[7149,7149],[7151,7153],[7212,7219],[7222,7223],[7376,7378],[7380,7392],[7394,7400],[7405,7405],[7412,7412],[7416,7417],[7616,7673],[7675,7679],[8203,8207],[8234,8238],[8288,8292],[8294,8303],[8400,8432],[11503,11505],[11647,11647],[11744,11775],[12330,12333],[12441,12442],[42607,42610],[42612,42621],[42654,42655],[42736,42737],[43010,43010],[43014,43014],[43019,43019],[43045,43046],[43204,43205],[43232,43249],[43263,43263],[43302,43309],[43335,43345],[43392,43394],[43443,43443],[43446,43449],[43452,43453],[43493,43493],[43561,43566],[43569,43570],[43573,43574],[43587,43587],[43596,43596],[43644,43644],[43696,43696],[43698,43700],[43703,43704],[43710,43711],[43713,43713],[43756,43757],[43766,43766],[44005,44005],[44008,44008],[44013,44013],[64286,64286],[65024,65039],[65056,65071],[65279,65279],[65529,65531]],i=[[66045,66045],[66272,66272],[66422,66426],[68097,68099],[68101,68102],[68108,68111],[68152,68154],[68159,68159],[68325,68326],[68900,68903],[69446,69456],[69633,69633],[69688,69702],[69759,69761],[69811,69814],[69817,69818],[69821,69821],[69837,69837],[69888,69890],[69927,69931],[69933,69940],[70003,70003],[70016,70017],[70070,70078],[70089,70092],[70191,70193],[70196,70196],[70198,70199],[70206,70206],[70367,70367],[70371,70378],[70400,70401],[70459,70460],[70464,70464],[70502,70508],[70512,70516],[70712,70719],[70722,70724],[70726,70726],[70750,70750],[70835,70840],[70842,70842],[70847,70848],[70850,70851],[71090,71093],[71100,71101],[71103,71104],[71132,71133],[71219,71226],[71229,71229],[71231,71232],[71339,71339],[71341,71341],[71344,71349],[71351,71351],[71453,71455],[71458,71461],[71463,71467],[71727,71735],[71737,71738],[72148,72151],[72154,72155],[72160,72160],[72193,72202],[72243,72248],[72251,72254],[72263,72263],[72273,72278],[72281,72283],[72330,72342],[72344,72345],[72752,72758],[72760,72765],[72767,72767],[72850,72871],[72874,72880],[72882,72883],[72885,72886],[73009,73014],[73018,73018],[73020,73021],[73023,73029],[73031,73031],[73104,73105],[73109,73109],[73111,73111],[73459,73460],[78896,78904],[92912,92916],[92976,92982],[94031,94031],[94095,94098],[113821,113822],[113824,113827],[119143,119145],[119155,119170],[119173,119179],[119210,119213],[119362,119364],[121344,121398],[121403,121452],[121461,121461],[121476,121476],[121499,121503],[121505,121519],[122880,122886],[122888,122904],[122907,122913],[122915,122916],[122918,122922],[123184,123190],[123628,123631],[125136,125142],[125252,125258],[917505,917505],[917536,917631],[917760,917999]],o=[[4352,4447],[8986,8987],[9001,9002],[9193,9196],[9200,9200],[9203,9203],[9725,9726],[9748,9749],[9800,9811],[9855,9855],[9875,9875],[9889,9889],[9898,9899],[9917,9918],[9924,9925],[9934,9934],[9940,9940],[9962,9962],[9970,9971],[9973,9973],[9978,9978],[9981,9981],[9989,9989],[9994,9995],[10024,10Line truncated
+//# sourceMappingURL=addon-unicode11.js.map

tests/DodoSSH.Client.Terminal.Tests/Renderer/renderer-harness.js

@@ -203,12 +203,22 @@ function FakeTerminal(options) {
   this.focuses = 0;
   this.disposed = false;
   this.pane = null;
+  this.addons = [];
+
+  // The proposed unicode API, at xterm's own default, so a test can see which table the page picked.
+  this.unicode = { activeVersion: '6' };
 
   terminals.push(this);
 }
 
-FakeTerminal.prototype.loadAddon = function () {};
+FakeTerminal.prototype.loadAddon = function (addon) { this.addons.push(addon); };
-FakeTerminal.prototype.open = function (pane) { this.pane = pane; };
+FakeTerminal.prototype.open = function (pane) {
+  this.pane = pane;
+
+  // Taken at open() because that is when the first frame is laid out: a table chosen afterwards would leave
+  // whatever was drawn first measured with the old one.
+  this.unicodeAtOpen = this.unicode.activeVersion;
+};
 FakeTerminal.prototype.attachCustomKeyEventHandler = function (handler) { this.keys = handler; };
 FakeTerminal.prototype.onData = function (handler) { this.data = handler; };
 FakeTerminal.prototype.onResize = function (handler) { this.resized = handler; };
@@ -242,6 +252,12 @@ var FitAddon = {
   },
 };
 
+var Unicode11Addon = {
+  Unicode11Addon: function () {
+    this.kind = 'unicode11';
+  },
+};
+
 var WebglAddon = {
   WebglAddon: function () {
     this.onContextLoss = function () {};
@@ -489,6 +505,16 @@ function typed(sessionId) {
   return terminalFor(sessionId).inputs.join('|');
 }
 
+/** Which character-width table the pane was opened with. */
+function unicodeAtOpen(sessionId) {
+  return terminalFor(sessionId).unicodeAtOpen;
+}
+
+/** Whether that pane loaded the Unicode 11 addon. */
+function hasUnicode11(sessionId) {
+  return terminalFor(sessionId).addons.some(function (addon) { return addon.kind === 'unicode11'; });
+}
+
 /** How many times that pane has been asked for the caret. */
 function focuses(sessionId) {
   return terminalFor(sessionId).focuses;

tests/DodoSSH.Client.Terminal.Tests/RendererPage.cs

@@ -94,6 +94,14 @@ internal sealed class RendererPage
     public string Typed(uint sessionId) =>
         engine.Evaluate(string.Create(CultureInfo.InvariantCulture, $"typed({sessionId})")).AsString();
 
+    /// <summary>Which character-width table that pane was opened with.</summary>
+    public string UnicodeAtOpen(uint sessionId) =>
+        engine.Evaluate(string.Create(CultureInfo.InvariantCulture, $"unicodeAtOpen({sessionId})")).AsString();
+
+    /// <summary>Whether that pane loaded the Unicode 11 addon.</summary>
+    public bool HasUnicode11(uint sessionId) =>
+        engine.Evaluate(string.Create(CultureInfo.InvariantCulture, $"hasUnicode11({sessionId})")).AsBoolean();
+
     /// <summary>How many times that session's pane has been asked for the caret.</summary>
     public int Focuses(uint sessionId) =>
         (int)engine.Evaluate(string.Create(CultureInfo.InvariantCulture, $"focuses({sessionId})")).AsNumber();

tests/DodoSSH.Client.Terminal.Tests/RendererUnicodeTests.cs

@@ -0,0 +1,43 @@
+namespace DodoSSH.Client.Terminal.Tests;
+
+/// <summary>
+/// The renderer page measures characters with Unicode 11's widths, so an emoji takes the two columns the
+/// remote counts for it.
+/// </summary>
+/// <remarks>
+/// What is asserted is that the page asks for the table, and asks before the first frame. Whether xterm then
+/// really draws 😀 across two cells is xterm's business, and the fake terminal in the harness is a stub with
+/// a memory, not an emulator; <c>docs/manual-checks.md</c> 1.13 is where a person looks at the real one.
+/// </remarks>
+public sealed class RendererUnicodeTests
+{
+    [Fact]
+    public void APane_LoadsTheUnicode11Addon()
+    {
+        var page = Opened();
+
+        page.HasUnicode11(7).ShouldBeTrue();
+    }
+
+    /// <remarks>
+    /// Before open() and not after it: a table chosen once the pane is open leaves whatever it drew first
+    /// measured with Unicode 6, which is the drift this exists to stop.
+    /// </remarks>
+    [Fact]
+    public void APane_IsOpenedWithUnicode11WidthsAlreadyActive()
+    {
+        var page = Opened();
+
+        page.UnicodeAtOpen(7).ShouldBe("11");
+    }
+
+    private static RendererPage Opened()
+    {
+        var page = RendererPage.Load();
+
+        page.Do("accept(0)");
+        page.Do("openSession(7)");
+
+        return page;
+    }
+}