Arrow key inputs are ignored when using MSYS bash with non-Mintty terminals

[yayugu]

I found the default MSYS ruby's irb is broken on Windows Terminal and VSCode terminal. It ignores all arrow keys (←→↑↓) so it couldn't move cursor or reuse prev-history.

After the investigation, I found Reline's logic to decide the IO gate (Reline::ANSI or Reline::Windows) is incorrect. It only correctly detect msys_tty? == true when using Mintty.

Step to reproduce

• Windows 11 25H2

1. Install MSYS2
2. Setup MSYS2 for Windows Terminal (see the official doc https://www.msys2.org/docs/terminals/ )
3. pacman -Sy ruby
4. Run irb and try to use arrow keys

Possible solutions

Here are some ideas (I'm not familiar with neither terminal techs nor windows system, though):

1. Respect $TERM. That env var (like TERM="xterm-256color") is only available on UNIX-like console. Probably safe to use Reline::ANSI instead of Reline::Windows.
2. Reline::Windows also allows unix-style arrow keys input. I'm not sure about the difference. But it seems they aren't conflict and safe to allow both.
3. A new env var for switching behavior. This is not a great solution but, it's good workaround if we can define and pass the env var to change the behavior with RELINE_IO=dump/ansi/windows

---

Debug script:

#!/usr/bin/env ruby
# frozen_string_literal: true

require "rbconfig"

def print_env
  puts "Ruby"
  puts "  RUBY_VERSION=#{RUBY_VERSION}"
  puts "  RUBY_PLATFORM=#{RUBY_PLATFORM}"
  puts "  ruby=#{RbConfig.ruby}"
  puts "  host_os=#{RbConfig::CONFIG["host_os"]}"
  puts "  target_os=#{RbConfig::CONFIG["target_os"]}"
  puts
  puts "TTY"
  puts "  STDIN.tty?=#{STDIN.tty?}"
  puts "  STDOUT.tty?=#{STDOUT.tty?}"
  puts
  puts "Environment"
  %w[
    TERM
    COLORTERM
    MSYSTEM
    MSYS
    WT_SESSION
    WT_PROFILE_ID
    ConEmuPID
    MINTTY_SHORTCUT
    SHELL
  ].each do |name|
    puts "  #{name}=#{ENV[name].inspect}"
  end
end

def printable_byte(byte)
  if byte.between?(32, 126)
    byte.chr
  else
    case byte
    when 7 then "BEL"
    when 8 then "BS"
    when 9 then "TAB"
    when 10 then "LF"
    when 13 then "CR"
    when 27 then "ESC"
    when 127 then "DEL"
    else "."
    end
  end
end

def run_raw
  require "io/console"

  print_env
  puts
  puts "Raw byte mode"
  puts "  Press keys to see bytes. Try Up, Down, Left, Right, Backspace, Enter."
  puts "  Press Ctrl-C to quit."
  puts

  STDOUT.sync = true
  buffer = []

  STDIN.raw do |io|
    loop do
      byte = io.getbyte
      break if byte == 3

      buffer << byte
      printf "%-5s 0x%02X  %3d  %s\n", "byte", byte, byte, printable_byte(byte)

      if byte == 65 && buffer[-3, 3] == [27, 91, 65]
        puts "      recognized sequence: ESC [ A  (cursor up)"
      elsif byte == 66 && buffer[-3, 3] == [27, 91, 66]
        puts "      recognized sequence: ESC [ B  (cursor down)"
      elsif byte == 67 && buffer[-3, 3] == [27, 91, 67]
        puts "      recognized sequence: ESC [ C  (cursor right)"
      elsif byte == 68 && buffer[-3, 3] == [27, 91, 68]
        puts "      recognized sequence: ESC [ D  (cursor left)"
      elsif byte == 65 && buffer[-3, 3] == [27, 79, 65]
        puts "      recognized sequence: ESC O A  (application cursor up)"
      elsif byte == 66 && buffer[-3, 3] == [27, 79, 66]
        puts "      recognized sequence: ESC O B  (application cursor down)"
      elsif byte == 67 && buffer[-3, 3] == [27, 79, 67]
        puts "      recognized sequence: ESC O C  (application cursor right)"
      elsif byte == 68 && buffer[-3, 3] == [27, 79, 68]
        puts "      recognized sequence: ESC O D  (application cursor left)"
      end
    end
  end
end

def run_timing
  require "io/console"

  print_env
  puts
  puts "Raw byte timing mode"
  puts "  Press Up/Down a few times. Press Ctrl-C to quit."
  puts "  This checks whether ESC and the following bytes arrive with suspicious gaps."
  puts

  STDOUT.sync = true
  previous = Process.clock_gettime(Process::CLOCK_MONOTONIC)

  STDIN.raw do |io|
    loop do
      byte = io.getbyte
      now = Process.clock_gettime(Process::CLOCK_MONOTONIC)
      delta_ms = ((now - previous) * 1000).round(3)
      previous = now

      break if byte == 3

      printf "dt=%9.3f ms  0x%02X  %3d  %s\r\n", delta_ms, byte, byte, printable_byte(byte)
    end
  end
end

def run_stty
  print_env
  puts
  puts "stty -a"
  system("stty", "-a")
end

def run_gate
  require "reline"
  require "reline/io/windows"

  print_env
  puts
  puts "Reline IO gate"

  io = Reline::IO.decide_io_gate
  puts "  decide_io_gate.class=#{io.class}"
  puts "  win?=#{io.win?}" if io.respond_to?(:win?)

  if io.respond_to?(:msys_tty?)
    begin
      puts "  msys_tty?=#{io.msys_tty?}"
    rescue StandardError => e
      puts "  msys_tty? error=#{e.class}: #{e.message}"
    end
  else
    puts "  msys_tty?=(method not available)"
  end

  begin
    windows = Reline::Windows.new
    puts
    puts "Windows stdin handle details"
    print_windows_handle_details(windows)
  rescue StandardError => e
    puts "  Could not inspect Windows handle: #{e.class}: #{e.message}"
  end

  begin
    require "reline/io/ansi"
    puts
    puts "Reline::ANSI default cursor bindings"
    puts "  Up ESC [ A => #{Reline::ANSI::ANSI_CURSOR_KEY_BINDINGS["A"].inspect}"
    puts "  Down ESC [ B => #{Reline::ANSI::ANSI_CURSOR_KEY_BINDINGS["B"].inspect}"
  rescue StandardError => e
    puts "  Could not inspect ANSI bindings: #{e.class}: #{e.message}"
  end
end

def print_windows_handle_details(_windows)
  api = Reline::Windows::Win32API
  get_std_handle = api.new("kernel32", "GetStdHandle", ["L"], "L")
  get_file_type = api.new("kernel32", "GetFileType", ["L"], "L")
  get_file_info = api.new("kernel32", "GetFileInformationByHandleEx", ["L", "I", "P", "L"], "I")

  std_input_handle = -10
  file_type_disk = 0x0001
  file_type_char = 0x0002
  file_type_pipe = 0x0003
  file_name_info = 2

  handle = get_std_handle.call(std_input_handle)
  file_type = get_file_type.call(handle)
  type_name = case file_type
              when file_type_disk then "FILE_TYPE_DISK"
              when file_type_char then "FILE_TYPE_CHAR"
              when file_type_pipe then "FILE_TYPE_PIPE"
              else "unknown"
              end

  puts "  GetStdHandle(STD_INPUT_HANDLE)=0x#{handle.to_s(16)}"
  puts "  GetFileType(stdin)=0x#{file_type.to_s(16)} #{type_name}"

  unless file_type == file_type_pipe
    puts "  pipe_name=(not a pipe, so Reline::Windows#msys_tty? returns false)"
    return
  end

  bufsize = 1024
  buffer = "\0" * bufsize
  result = get_file_info.call(handle, file_name_info, buffer, bufsize - 2)
  if result == 0
    puts "  pipe_name=(GetFileInformationByHandleEx failed)"
    return
  end

  length = buffer[0, 4].unpack1("L")
  name = buffer[4, length].encode(Encoding::UTF_8, Encoding::UTF_16LE, invalid: :replace)
  puts "  pipe_name=#{name.inspect}"
  puts "  matches_msys_pty?=#{!!(name =~ /(msys-|cygwin-).*-pty/)}"
end

def clean_inputrc_for_reline
  <<~INPUTRC
    set editing-mode emacs
    set keymap emacs
    set keyseq-timeout 0
    "\\e[A": ed-prev-history
    "\\e[B": ed-next-history
    "\\e[C": ed-next-char
    "\\e[D": ed-prev-char
    "\\eOA": ed-prev-history
    "\\eOB": ed-next-history
    "\\eOC": ed-next-char
    "\\eOD": ed-prev-char
  INPUTRC
end

def clean_inputrc_for_readline
  <<~INPUTRC
    set editing-mode emacs
    set keymap emacs
    set keyseq-timeout 0
    "\\e[A": previous-history
    "\\e[B": next-history
    "\\e[C": forward-char
    "\\e[D": backward-char
    "\\eOA": previous-history
    "\\eOB": next-history
    "\\eOC": forward-char
    "\\eOD": backward-char
  INPUTRC
end

def with_clean_inputrc(kind)
  require "tmpdir"

  Dir.mktmpdir("ruby-inputrc-") do |dir|
    inputrc = File.join(dir, "inputrc")
    content = kind == :readline ? clean_inputrc_for_readline : clean_inputrc_for_reline
    File.write(inputrc, content)
    ENV["INPUTRC"] = inputrc
    puts "Using temporary INPUTRC=#{inputrc}"
    puts content
    puts
    yield
  end
end

def run_reline
  require "reline"

  print_env
  puts
  puts "Reline mode"
  puts "  Type a few lines, then try Up/Down history."
  puts "  Press Ctrl-D on an empty line to quit."
  puts

  i = 0
  while (line = Reline.readline("reline#{i += 1}> ", true))
    p line
  end
end

def run_reline_clean
  with_clean_inputrc(:reline) do
    load_reline_and_run
  end
end

def load_reline_and_run
  require "reline"

  print_env
  begin
    core = Reline.respond_to?(:core) ? Reline.core : nil
    config = core&.respond_to?(:config) ? core.config : nil
    if config
      puts
      puts "Reline config"
      %i[keyseq_timeout editing_mode inputrc_path].each do |name|
        value =
          if config.respond_to?(name)
            config.public_send(name)
          elsif config.instance_variable_defined?(:"@#{name}")
            config.instance_variable_get(:"@#{name}")
          else
            "(unavailable)"
          end
        puts "  #{name}=#{value.inspect}"
      end
    end
  rescue StandardError => e
    warn "Could not inspect Reline config: #{e.class}: #{e.message}"
  end

  puts
  puts "Reline mode"
  puts "  Type a few lines, then try Up/Down history."
  puts "  Press Ctrl-D on an empty line to quit."
  puts

  i = 0
  while (line = Reline.readline("reline#{i += 1}> ", true))
    p line
  end
end

def run_readline
  begin
    require "readline"
  rescue LoadError => e
    warn "Could not load readline: #{e.class}: #{e.message}"
    exit 2
  end

  print_env
  puts
  puts "Readline mode"
  puts "  Readline::VERSION=#{Readline.const_defined?(:VERSION) ? Readline::VERSION : "(no VERSION constant)"}"
  puts "  Type a few lines, then try Up/Down history."
  puts "  Press Ctrl-D on an empty line to quit."
  puts

  i = 0
  while (line = Readline.readline("readline#{i += 1}> ", true))
    p line
  end
end

def run_readline_clean
  with_clean_inputrc(:readline) do
    run_readline
  end
end

def usage
  warn <<~USAGE
    Usage:
      ruby terminal_input_probe.rb env
      ruby terminal_input_probe.rb raw
      ruby terminal_input_probe.rb timing
      ruby terminal_input_probe.rb stty
      ruby terminal_input_probe.rb gate
      ruby terminal_input_probe.rb reline
      ruby terminal_input_probe.rb reline-clean
      ruby terminal_input_probe.rb readline
      ruby terminal_input_probe.rb readline-clean

    Suggested order:
      1. env
      2. raw
      3. timing
      4. gate
      5. reline-clean
      6. readline-clean
  USAGE
end

case ARGV.fetch(0, "help")
when "env"
  print_env
when "raw"
  run_raw
when "timing"
  run_timing
when "stty"
  run_stty
when "gate"
  run_gate
when "reline"
  load_reline_and_run
when "reline-clean"
  run_reline_clean
when "readline"
  run_readline
when "readline-clean"
  run_readline_clean
else
  usage
  exit 1
end

Windows Terminal:

Ruby
  RUBY_VERSION=3.4.8
  RUBY_PLATFORM=x86_64-cygwin
  ruby=/usr/bin/ruby.exe
  host_os=cygwin
  target_os=cygwin

TTY
  STDIN.tty?=true
  STDOUT.tty?=true

Environment
  TERM="xterm-256color"
  COLORTERM=nil
  MSYSTEM="MSYS"
  MSYS="winsymlinks:nativestrict"
  WT_SESSION="3be8b3a2-0619-4392-af71-591b5bfc38e5"
  WT_PROFILE_ID="{71160544-14d8-4194-af25-d05feeac7233}"
  ConEmuPID=nil
  MINTTY_SHORTCUT=nil
  SHELL="/usr/bin/zsh"

Reline IO gate
  decide_io_gate.class=Reline::Windows
  win?=true
  msys_tty?=false

Windows stdin handle details
  GetStdHandle(STD_INPUT_HANDLE)=0x1d8
  GetFileType(stdin)=0x2 FILE_TYPE_CHAR
  pipe_name=(not a pipe, so Reline::Windows#msys_tty? returns false)

Reline::ANSI default cursor bindings
  Up ESC [ A => [:ed_prev_history, {}]
  Down ESC [ B => [:ed_next_history, {}]

Mintty:

$ ruby terminal_input_probe.rb gate
Ruby
  RUBY_VERSION=3.4.8
  RUBY_PLATFORM=x86_64-cygwin
  ruby=/usr/bin/ruby.exe
  host_os=cygwin
  target_os=cygwin

TTY
  STDIN.tty?=true
  STDOUT.tty?=true

Environment
  TERM="xterm-256color"
  COLORTERM=nil
  MSYSTEM="MSYS"
  MSYS=nil
  WT_SESSION=nil
  WT_PROFILE_ID=nil
  ConEmuPID=nil
  MINTTY_SHORTCUT=nil
  SHELL="/usr/bin/bash"

Reline IO gate
  decide_io_gate.class=Reline::ANSI
  win?=false
  msys_tty?=(method not available)

Windows stdin handle details
  GetStdHandle(STD_INPUT_HANDLE)=0x4b4
  GetFileType(stdin)=0x3 FILE_TYPE_PIPE
  pipe_name="\\msys-dd50a72ab4668b33-pty0-from-master"
  matches_msys_pty?=true

Reline::ANSI default cursor bindings
  Up ESC [ A => [:ed_prev_history, {}]
  Down ESC [ B => [:ed_next_history, {}]