shell: all commands are now interactive: interactive output, emacs -nw works

needs our Clesh fork.

Makefile

@@ -54,6 +54,11 @@ ql-deps: $(QLDIR)/asdf
 	$(call git-clone-pull,https://github.com/slburson/fset)
 
 
+	# updated Clesh for a shell pass-through that handles all shell commands interactively.
+	# So we now see the output in real time (instead of at the end of the execution),
+	# and commands like "emacs -nw" now work, in addition of sudo, vim or htop that were handled separately.
+	git clone https://github.com/lisp-maintainers/clesh ~/quicklisp/local-projects/clesh
+
 # Install some system dependencies.
 debian-deps:
 	apt-get install -y libinotifytools0

README.md

@@ -342,20 +342,21 @@ CIEL ships a terminal REPL for the terminal which is more user friendly than the
 - it handles errors gracefully: you are not dropped into the debugger
   and its sub-REPL, you simply see the error message.
 - it has optional **syntax highlighting**.
-- it has an optional **lisp critic** that scans the code you enter at
-  the REPL for instances of bad practices.
 - it has a **shell pass-through**: try `!ls`.
-  - it runs **interactive commands**: try `!htop`, `!vim test.lisp`, `!emacsclient test.lisp` or `!env FOO=BAR sudo -i powertop`.
+  - you can mix and match shell and Lisp: try `!echo ?(+ 1/3 1/3)` (look, a fraction)
+  - it runs **interactive commands**: try `!htop`, `!vim test.lisp`, `!emacs -nw test.lisp` or `!env FOO=BAR sudo -i powertop`.
 - it has **documentation lookup** shorthands: use `:doc symbol` or `?`
   after a symbol to get its documentation: `ciel-user> (dict ?`.
 - it has **developer friendly** macros: use `(printv code)` for an
   annotated trace output.
-- it integrates the **lisp critic**.
+- it has an optional **lisp critic** that scans the code you enter at
+  the REPL for instances of bad practices.
 - and it defines some more helper commands.
+- it works on Slime (to a certain extent)
 
 The CIEL terminal REPL loads the `~/.cielrc` init file at start-up if present. Don't load it with `--no-userinit`.
 
-See more in [*the documentation*](https://ciel-lang.github.io/CIEL/#/).
+See more in [*the documentation*](https://ciel-lang.github.io/CIEL/#/repl).
 
 > [!NOTE]
 > Our terminal readline REPL does NOT replace a good Common Lisp editor. You have more choices than Emacs. Check them out! https://lispcookbook.github.io/cl-cookbook/editor-support.html

docs/repl.md

@@ -5,12 +5,15 @@ CIEL's REPL is more user friendly than the default SBCL one. In particular:
 -  it has readline capabilities, meaning that the arrow keys work by default (wouhou!) and there is a persistent history, like in any shell.
 -  it has **multiline input**.
 -  it has **TAB completion**.
+  - including for files (after a bracket) and binaries in the PATH.
 -  it handles errors gracefully: you are not dropped into the debugger and its sub-REPL, you simply see the error message.
 -  it has optional **syntax highlighting**.
-
+- it has a **shell pass-through**: try `!ls` (also available in Slime)
-- it has a **shell pass-through**: try `!ls` (available in the `ciel-user` package)
+  - you can mix and match shell and Lisp: try `!echo ?(+ 1/3 1/3)` (look, a fraction)
-  - it runs **interactive commands**: try `!htop`, `!vim test.lisp`, `!emacsclient test.lisp` or `!env FOO=BAR sudo -i powertop`.
+  - it runs **interactive commands**: try `!htop`, `!vim test.lisp`, `!emacs -nw test.lisp` or `!env FOO=BAR sudo -i powertop`.
 - it has a quick **edit and load file** command: calling `%edit file.lisp` will open the file with the editor of the EDITOR environment variable. When you close it, the file is loaded and evaluated.
+- it has an optional **lisp critic** that scans the code you enter at
+  the REPL for instances of bad practices.
 
 -  it defines more **helper commands**:
 
@@ -26,7 +29,7 @@ CIEL's REPL is more user friendly than the default SBCL one. In particular:
 
 Our REPL is adapted from [sbcli](https://github.com/hellerve/sbcli). See also [cl-repl](https://github.com/koji-kojiro/cl-repl/), that has an interactive debugger.
 
-> Note: a shell interface doesn't replace a good development environment. See this [list of editors for Common Lisp](https://lispcookbook.github.io/cl-cookbook/editor-support.html): Emacs, Vim, Atom, VSCode, SublimeText, Jupyter Notebooks and more.
+> Note: a shell interface doesn't replace a good development environment. See this [list of editors for Common Lisp](https://lispcookbook.github.io/cl-cookbook/editor-support.html): Emacs, Vim, Atom, VSCode, Intellij, SublimeText, Jupyter Notebooks and more.
 
 ## Quick documentation lookup
 
@@ -39,58 +42,53 @@ ciel-user> %doc dict
 ciel-user> (dict ?
 ```
 
-## Shell pass-through
+## Shell pass-through with "!"
 
 Use `!` to send a shell command:
 
 ```
 !ls
-Makefile
+!sudo emacs -nw /etc/
-README.org
-repl.lisp
-repl-utils.lisp
-src
-...
-
-!pwd
-/home/vindarel/projets/ciel
 ```
 
-Use square brackets `[...]` to write a shell script, and use `$` inside it to escape to lisp:
+### Mixing Lisp code with "?"
 
-```lisp
+You can mix shell commands and Lisp code. The `?` character is the
-(dotimes (i 7) (princ [echo ?i]))
+"lisp escape". For example:
-```
 
-The result is concatenated into a string and printed on stdout.
+    * !echo ?(+ 2 3)
 
-This feature is only available by default in CIEL's REPL, not on the
+or:
+
+    * (defun factorial (x) (if (zerop x) 1 (* x (factorial (1- x)))))
+    * !echo "fact 3: " ?(factorial 3)
+
+Escape the "?" with "\?" to write it in a shell command.
+
+### Shell commands in Slime and limitations
+
+The "!" shell pass-through is available by default in CIEL's REPL, not in the
 CIEL-USER package. To enable it yourself, do:
 
-      (ciel:enable-shell-passthrough)
+      CIEL-USER> (enable-shell-passthrough)
 
-But, some programs are **visual**, or interactive, because they have an ncurses or similar interface. They need
+There are differences on how shell commands are handled in the terminal REPL and in Slime.
-to be run in their own terminal window. CIEL recognizes a few (`vim`,
-`htop`, `man`… see `*visual-commands*`) and runs them in the first terminal emulator found on
-the system: `terminator`, `xterm`, `gnome-terminal`, Emacs' `vterm` (with emacsclient) or your own.
 
-So, you can run a command similar to this one:
+All shell commands in the terminal are run interactively. You can see
+the program output as it goes. In Emacs and Slime, the commands are
+run *synchronously*, the output (and error output) is captured and
+displayed when the command is finished.
 
-    ENV=env sudo htop
+In the terminal REPL, you can use `sudo`, `emacs -nw` and other visual
+and interactive commands, but not in Slime.
 
-and it will open in a new terminal (hint: a visual command doesn't require the `!` prefix).
+> Note: the shell-passthrough feature is experimental.
 
-To use your terminal emulator of choice, do:
+> Note: we encourage our users to use a good editor rather than a terminal!
 
-    (push "myterminal" *visual-terminal-emulator-choices*)
+We use our fork of the [Clesh](https://github.com/lisp-maintainers/clesh) library.
 
-> Note: this feature is experimental.
+See also [Lish](https://github.com/nibbula/lish/) and [SHCL](https://github.com/bradleyjensen/shcl) for more unholy union of (posix) shells and Common Lisp.
-
-> Note: we encourage our users to use Emacs rather than a terminal!
-
-We use the [Clesh](https://github.com/Neronus/clesh) library for the `!` shell passthrough.
-
-See also [SHCL](https://github.com/bradleyjensen/shcl) for a more unholy union of posix-shell and Common Lisp.
 
 
 ## Syntax highlighting
@@ -98,12 +96,11 @@ See also [SHCL](https://github.com/bradleyjensen/shcl) for a more unholy union o
 Syntax highlighting is off by default. To enable it, install [pygments](https://pygments.org/) and add this in your `~/.cielrc`:
 
 ```lisp
-(in-package :sbcli)
+(setf sbcli:*syntax-highlighting* t)
-(setf *syntax-highlighting* t)
 
 ;; and, optionally:
-;; (setf *pygmentize* "/path/to/pygmentize")
+;; (setf sbcli::*pygmentize* "/path/to/pygmentize")
-;; (setf *pygmentize-options* (list "-s" "-l" "lisp"))
+;; (setf sbcli::*pygmentize-options* (list "-s" "-l" "lisp"))
 ```
 
 You can also switch it on and off from the REPL:

repl.lisp

@@ -420,6 +420,13 @@ strings to match candidates against (for example in the form \"package:sym\")."
                   (select-completions "str:con" (list "str:containsp" "str:concat" "str:constant-case"))
                   :test #'string-equal)))
 
+(defun shell-passthrough-p (arg)
+  "Return t if arg (string) starts with \"!\".
+
+  This is used to offer custom TAB completion, not to launch shell commands.
+  The Clesh readtable is responsible of that."
+  (str:starts-with-p "!" arg))
+
 (defun complete-filename-p (text start end &key (line-buffer rl:*line-buffer*))
   "Return T if we should feed the tab completion candidates filenames, instead of the regular Lisp symbols.
   We answer yes when we are tab-completing a secord word on the prompt and a quote comes before it.
@@ -569,14 +576,12 @@ strings to match candidates against (for example in the form \"package:sym\")."
       ((str:ends-with-p " ?" text)
        (sbcli::symbol-documentation (last-nested-expr text)))
 
-      ;; Handle visual commands: run in their own terminal window.
+      ;; Interactive and visual shell command?
-      ;; XXX: we can do better, see
+      ;; They are now handled by Clesh.
-      ;; https://lispcookbook.github.io/cl-cookbook/os.html#running-interactive-and-visual-commands-htop
+      ;; When on a non "dumb" terminal, all shell commands are run interactively.
-      ((visual-command-p text)
-       (run-visual-command text))
 
-      ;; shell command? No need to check for a "!" in the input here,
+      ;; No need to check for a "!" in the input here,
-      ;; it's done with the clesh readtable later when handling lisp.
+      ;; it's done with the clesh readtable when handling lisp.
 
       ;; Default: run the lisp command (with the lisp-critic, the shell passthrough
       ;; and other add-ons).

shell-utils.lisp

@@ -3,12 +3,24 @@
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;;; Run visual / interactive / ncurses commands.
 ;;;
+;;; update <2024-09-04>: now all shell commands are run interactively.
+;;; It works for htop, vim, sudo, emacs -nw…
+;;;
+;;; except in Slime, where the interactivity doesn't work.
+;;; That means that the command is run synchronously and we see the output at once at the end.
+;;; So this code is meant to be used in Slime:
+;;; - guess a program is interactive
+;;; - run it on a new and dedicated terminal emulator (xterm or even Emacs' vterm).
+;;;
+;;; So,
 ;;; How to guess a program is interactive?
 ;;; We currently look from a hand-made list (à la Eshell).
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
+;; all this is unused as of <2024-09-04> for the terminal CIEL repl
+;; but is to be used to support visual commands in Slime.
 (defparameter *visual-commands*
-  '(;; "emacs -nw" ;; unsupported. In eshell, see the concept of visual-subcommands.
+  '(;; "emacs -nw" ;; unsupported in Slime, works on the terminal. In eshell, see the concept of visual-subcommands.
     "vim" "vi"
     "nano"
     "htop" "top"
@@ -17,10 +29,13 @@
     "lynx" "links" "mutt" "pine" "tin" "elm" "ncftp" "ncdu"
     "ranger"
     "mpv" "mplayer"
-    "ipython" "irb" "iex"               ;; TBC
+    "ipython" "irb" "iex" ;; TBC
     ;; last but not least
     "ciel-repl")
-  "List of visual/interactive/ncurses-based programs that will be run in their own terminal window.")
+  "List of visual/interactive/ncurses-based programs that will be run in their own terminal window.
+
+  Visual commands work by default in the terminal REPL.
+  This would be useful only in Slime.")
 
 (defun vterm-terminal (cmd)
   "Build a command (string) to send to emacsclient to open CMD with Emacs' vterm."
@@ -58,13 +73,6 @@
     (when arg
       (namestring (pathname-name arg)))))
 
-(defun shell-passthrough-p (arg)
-  "Return t if arg (string) starts with \"!\".
-
-  This is used to offer custom TAB completion, not to launch shell commands.
-  The Clesh readtable is responsible of that."
-  (str:starts-with-p "!" arg))
-
 (defun shell-command-wrapper-p (command)
   "Is this command (string) a shell wrapper? (such as sudo or env)