core-cli: backport more refactors from rewrite

Still a long way to go, but this introduces a few niceties for
debugging CLI failures:

+ The (extended) output of the last bin/doom command is now logged to
  ~/.emacs.d/.local/doom.log
+ If an error occurs, short backtraces are displayed whether or not you
  have debug mode on. The full backtrace is written to
  ~/.emacs.d/.local/doom.error.log.
+ bin/doom now aborts with a warning if:
  - The script itself or its parent directory is a symlink. It's fine if
    ~/.emacs.d is symlinked though.
  - Running bin/doom as root when your DOOMDIR isn't in /root/.
  - If you're sporting Emacs 26.1 (now handled in the elisp side rather
    than the /bin/sh shebang preamble).
+ If a 'doom sync' was aborted prematurely, you'll be warned that Doom
  was left in an inconsistent state and that you must run `doom sync`
  again.

May address #3746

bin/doom

@@ -1,126 +1,128 @@
 #!/usr/bin/env sh
-:; ( echo "$EMACS" | grep -q "term" ) && EMACS=emacs || EMACS=${EMACS:-emacs} # -*-emacs-lisp-*-
+:; set -e # -*- mode: emacs-lisp; lexical-binding: t -*-
-:; command -v $EMACS >/dev/null || { >&2 echo "Can't find emacs in your PATH"; exit 1; }
+:; ( echo "$EMACS" | grep -q "term" ) && EMACS=emacs || EMACS=${EMACS:-emacs}
-:; _VERSION=$($EMACS --version | head -n1)
+:; command -v "$EMACS" >/dev/null || { >&2 echo "Can't find emacs in your PATH"; exit 1; }
-:; case "$_VERSION" in *\ 2[0-5].[0-9]) echo "Detected Emacs $_VERSION"; echo "Doom only supports Emacs 26.1 and newer"; echo; exit 2 ;; esac
+:; export EMACSDIR="${EMACSDIR:-`dirname "$0"`/..}"
-:; _DOOMBASE="${EMACSDIR:-$(dirname "$0")/..}"
+:; export __DOOMPOST="${TMPDIR:-/tmp}/doom.sh"
-:; _DOOMPOST="$_DOOMBASE/.local/.doom.sh"
+:; __DOOMCODE=0
-:; $EMACS --no-site-file --script "$0" -- "$@"
+:; "$EMACS" --no-site-file --script "$0" -- "$@" || __DOOMCODE=$?
-:; CODE=$?
+:; [ $__DOOMCODE -eq 128 ] && { "$__DOOMPOST" "$0" "$@"; __DOOMCODE=$?; }
-:; [ -x "$_DOOMPOST" ] && "$_DOOMPOST" "$0" "$@"
+:; exit $__DOOMCODE
-:; exit $CODE
 
-;; CLI ops tend to eat a lot of memory. To speed it up, stave off the GC, but
+;; The garbage collector isn't important during CLI ops. A higher threshold
-;; not to `most-positive-fixnum' like we do in init.el; that's too high -- we
+;; makes it 15-30% faster, but set it too high and we risk spiralling memory
-;; don't want to intentionally leak memory.
+;; usage in longer sessions.
 (setq gc-cons-threshold 134217728)   ; 128mb
 
-(let* ((loaddir (file-name-directory (file-truename load-file-name)))
+;; Prioritize non-byte-compiled source files in non-interactive sessions to
-       (emacsdir (getenv "EMACSDIR"))
+;; prevent loading stale byte-code.
-       (user-emacs-directory
+(setq load-prefer-newer t)
-        (abbreviate-file-name
-         (if emacsdir
-             (file-name-as-directory emacsdir)
-           (expand-file-name "../" loaddir)))))
 
-  ;;
+;; Ensure Doom runs out of this file's parent directory, where Doom is
-  (load (expand-file-name "core/core.el" user-emacs-directory) nil t)
+;; presumably installed. EMACSDIR is set in the shell script preamble earlier in
+;; this file.
+(setq user-emacs-directory
+      (file-name-as-directory  ; ensure the trailing slash...
+       (expand-file-name (or (getenv "EMACSDIR") ""))))
 
-  ;; HACK Load `cl' and site files manually so we can stop them from polluting
+;; Handle some potential issues early
-  ;;      CLI logs with deprecation and file load messages.
+(when (version< emacs-version "26.1")
-  (quiet! (when (> emacs-major-version 26)
+  (error (concat "Detected Emacs %s (at %s).\n\n"
-            (require 'cl))
+                 "Doom only supports Emacs 26.1 and newer. 27.1 is highly recommended. A guide\n"
-          (load "site-start" t t))
+                 "to install a newer version of Emacs can be found at:\n\n  "
+                 (cond ((eq system-type 'darwin)
+                        "https://github.com/hlissner/doom-emacs/blob/develop/docs/getting_started.org#on-macos")
+                       ((memq system-type '(cygwin windows-nt ms-dos))
+                        "https://github.com/hlissner/doom-emacs/blob/develop/docs/getting_started.org#on-windows")
+                       ("https://github.com/hlissner/doom-emacs/blob/develop/docs/getting_started.org#on-linux"))
+                 "Aborting...")
+         emacs-version
+         (car command-line-args)))
 
-  (doom-log "Initializing Doom CLI")
+(unless (file-exists-p (expand-file-name "core/core.el" user-emacs-directory))
-  (require 'core-cli)
+  (error (concat "Couldn't find Doom Emacs in %S.\n\n"
+                 "This is likely because this script (or its parent directory) is a symlink.\n"
+                 "If you must use a symlink, you'll need to specify an EMACSDIR so Doom knows\n"
+                 "where to find itself. e.g.\n\n    "
+                 (if (string-match "/fish$" (getenv "SHELL"))
+                     "env EMACSDIR=~/.emacs.d doom"
+                   "EMACSDIR=~/.emacs.d doom sync")
+                 "\n\n"
+                 "Aborting...")
+         (abbreviate-file-name (file-truename user-emacs-directory))
+         (abbreviate-file-name load-file-name)))
 
-  (defcli! :main
+(when (and (equal (user-real-uid) 0)
-      ((help-p   ["-h" "--help"]    "Same as help command")
+           (not (file-in-directory-p user-emacs-directory "/root")))
-       (debug-p  ["-d" "--debug"]   "Turns on doom-debug-p (and debug-on-error)")
+  (error (concat "This script is running as root. This likely wasn't intentional and\n"
-       (yes-p    ["-y" "--yes"]     "Auto-accept all confirmation prompts")
+                 "will cause file permissions errors later if this Doom install is\n"
-       &optional command &rest args)
+                 "ever used on a non-root account.\n\n"
-    "A command line interface for managing Doom Emacs.
+                 "Aborting...")))
 
-Includes package management, diagnostics, unit tests, and byte-compilation.
+;; Load the heart of the beast and its CLI processing library
+(load (expand-file-name "core/core.el" user-emacs-directory) nil t)
+(require 'core-cli)
 
-This tool also makes it trivial to launch Emacs out of a different folder or
+;; Use our own home-grown debugger to display and log errors + backtraces.
-with a different private module."
+;; Control over its formatting is important, because Emacs produces
-    :bare t
+;; difficult-to-read debug information otherwise. By making its errors more
-    (when debug-p
+;; presentable (and storing them somewhere users can access them later) we go a
-      (setenv "DEBUG" "1")
+;; long way toward making it easier for users to write better bug reports.
-      (setq doom-debug-p t)
+(setq debugger #'doom-cli--debugger
-      (print! (info "Debug mode on")))
+      debug-on-error t
-    (when yes-p
+      debug-ignored-errors nil)
-      (setenv "YES" "1")
-      (setq doom-auto-accept t)
-      (print! (info "Auto-yes on")))
-    (when help-p
-      (when command
-        (push command args))
-      (setq command "help"))
 
-    (when (equal (user-real-uid) 0)
+;; HACK Load `cl' and site files manually to prevent polluting logs and stdout
-      (print!
+;;      with deprecation and/or file load messages.
-       (concat "WARNING: This script is running as root. This likely wasn't intentional, and\n"
+(quiet! (if EMACS27+ (require 'cl))
-               "is unnecessary to use this script. This will cause file permissions errors\n"
+        (load "site-start" t t))
-               "later if you use this Doom installation on a non-root account.\n"))
-      (unless (or doom-auto-accept (y-or-n-p "Continue anyway?"))
-        (user-error "Aborted")))
 
-    ;; Load any the user's private init.el or any cli.el files in modules. This
+(kill-emacs
-    ;; gives the user (and modules) an opportunity to define their own CLI
+ (pcase
-    ;; commands, or to customize the CLI to better suit them.
+     ;; Process the arguments passed to this script. `doom-cli-execute' should
-    (load! doom-module-init-file doom-private-dir t)
+     ;; return a boolean, integer (error code) or throw an 'exit event, which we
-    (maphash (doom-module-loader doom-cli-file) doom-modules)
+     ;; handle specially.
-    (load! doom-cli-file doom-private-dir t)
+     (apply #'doom-cli-execute :doom (cdr (member "--" argv)))
-    (run-hooks 'doom-cli-pre-hook)
+   ;; Any non-zero integer is treated as an error code.
-
+   ((and (pred integerp) code) code)
-    (let (print-level print-gensym print-length)
+   ;; If, instead, we were given a list or string, copy these as shell script
-      (condition-case e
+   ;; commands to a temp script file which this script will execute after this
-          (if (null command)
+   ;; session finishes. Also accepts special keywords, like `:restart', to rerun
-              (doom-cli-execute "help")
+   ;; the current command.
-            (let ((start-time (current-time)))
+   ((and (or (pred consp)
-              (and (doom-cli-execute command args)
+             (pred stringp)
-                   (progn (run-hooks 'doom-cli-post-hook) t)
+             (pred keywordp))
-                   (print! (success "Finished! (%.4fs)")
+         command)
-                           (float-time
+    (let ((script (doom-path (getenv "__DOOMPOST")))
-                            (time-subtract (current-time)
+          (coding-system-for-write 'utf-8-unix)
-                                           start-time))))))
+          (coding-system-for-read  'utf-8-unix))
-        (user-error
+      (with-temp-file script
-         (print! (error "%s\n") (error-message-string e))
+        (insert "#!/usr/bin/env sh\n"
-         (print! (yellow "See 'doom help %s' for documentation on this command.") (car args))
+                "_postscript() {\n"
-         (error "")) ; Ensure non-zero exit code
+                "  rm -f " (shell-quote-argument script) "\n  "
-        ((debug error)
+                (cond ((eq command :restart)
-         (print! (error "There was an unexpected error:"))
+                       "$@")
-         (print-group!
+                      ((stringp command)
-          (print! "%s %s" (bold "Type:") (car e))
+                       command)
-          (print! (bold "Message:"))
+                      ((string-join
-          (print-group!
+                        (if (listp (car-safe command))
-           (print! "%s" (get (car e) 'error-message)))
+                            (cl-loop for line in (doom-enlist command)
-          (print! (bold "Data:"))
+                                     collect (mapconcat #'shell-quote-argument (remq nil line) " "))
-          (print-group!
+                          (list (mapconcat #'shell-quote-argument (remq nil command) " ")))
-           (if (cdr e)
+                        "\n  ")))
-               (dolist (item (cdr e))
+                "\n}\n"
-                 (print! "%S" item))
+                (save-match-data
-             (print! "n/a")))
+                  (cl-loop for env in process-environment
-          (when (and (bound-and-true-p straight-process-buffer)
+                           if (string-match "^\\([a-zA-Z0-9_]+\\)=\\(.+\\)$" env)
-                     (string-match-p (regexp-quote straight-process-buffer)
+                           concat (format "%s=%s \\\n"
-                                     (error-message-string e)))
+                                          (match-string 1 env)
-            (print! (bold "Straight output:"))
+                                          (shell-quote-argument (match-string 2 env)))))
-            (print-group! (print! "%s" (straight--process-get-output)))))
+                (format "PATH=\"%s%s$PATH\" \\\n" (concat doom-emacs-dir "bin/") path-separator)
-         (unless debug-on-error
+                "_postscript $@\n"))
-           (terpri)
+      (set-file-modes script #o700))
-           (print!
+    ;; Error code 128 is special: it means run the post-script after this
-            (concat "Run the command again with the -d (or --debug) switch to enable debug\n"
+    ;; session ends.
-                    "mode and (hopefully) generate a backtrace from this error:\n"
+    128)
-                    "\n  %s\n\n"
+   ;; Anything else (e.g. booleans) is treated as a successful run. Yes, a `nil'
-                    "If you file a bug report, please include it!")
+   ;; indicates a successful run too!
-            (string-join (append (list (file-name-nondirectory load-file-name) "-d" command)
+   (_ 0)))
-                                 args)
-                         " "))
-           ;; Ensure non-zero exit code
-           (error ""))))))
-
-  (doom-cli-execute :main (cdr (member "--" argv)))
-  (setq argv nil))

core/autoload/config.el

@@ -98,7 +98,7 @@ Runs `doom-reload-hook' afterwards."
 
 ;;;###autoload
 (defun doom/reload-autoloads ()
-  "Reload only `doom-autoload-file' and `doom-package-autoload-file'.
+  "Reload only `doom-autoloads-file' and `doom-package-autoload-file'.
 
 This is much faster and safer than `doom/reload', but not as comprehensive. This
 reloads your package and module visibility, but does not install new packages or

core/autoload/files.el

@@ -146,8 +146,9 @@ If COOKIE doesn't exist, return NULL-VALUE."
     (insert-file-contents file nil 0 256)
     (if (re-search-forward (format "^;;;###%s " (regexp-quote (or cookie "if")))
                            nil t)
-        (let ((load-file-name file))
+        (or (let ((load-file-name file))
-          (eval (sexp-at-point) t))
+              (eval (sexp-at-point) t))
+            null-value)
       null-value)))
 
 ;;;###autoload

core/autoload/output.el

@@ -254,8 +254,12 @@ DEST can be one or more of `standard-output', a buffer, a file"
                (insert-char out))
              (send-string-to-terminal (char-to-string out)))))
      (letf! (defun message (msg &rest args)
-              (princ (apply #'format msg args))
+              (with-current-buffer log-buffer
-              (terpri))
+                (print-group!
+                 (insert (doom--format (apply #'format msg args)) "\n")))
+              (if doom-debug-p
+                  (doom--print (doom--format (apply #'format msg args)))
+                (apply message msg args)))
        (unwind-protect
            ,(macroexp-progn body)
          (with-current-buffer log-buffer

core/cli/autoloads.el

@@ -14,7 +14,7 @@ one wants that.")
     auto-mode-alist
     interpreter-mode-alist
     Info-directory-list)
-  "A list of variables to be cached in `doom-autoload-file'.")
+  "A list of variables to be cached in `doom-autoloads-file'.")
 
 (defvar doom-autoloads-files ()
   "A list of additional files or file globs to scan for autoloads.")
@@ -26,7 +26,7 @@ one wants that.")
 (defun doom-autoloads-reload (&optional file)
   "Regenerates Doom's autoloads and writes them to FILE."
   (unless file
-    (setq file doom-autoload-file))
+    (setq file doom-autoloads-file))
   (print! (start "(Re)generating autoloads file..."))
   (print-group!
    (cl-check-type file string)

core/cli/sync.el

@@ -0,0 +1,56 @@
+;;; core/cli/sync.el -*- lexical-binding: t; -*-
+
+(defcli! (sync s)
+  ((no-envvar-p ["-e"] "Don't regenerate the envvar file")
+   (no-elc-p    ["-c"] "Don't recompile config")
+   (update-p    ["-u"] "Update installed packages after syncing")
+   (purge-p     ["-p" "--prune"] "Purge orphaned package repos & regraft them"))
+  "Synchronize your config with Doom Emacs.
+
+This is the equivalent of running autoremove, install, autoloads, then
+recompile. Run this whenever you:
+
+  1. Modify your `doom!' block,
+  2. Add or remove `package!' blocks to your config,
+  3. Add or remove autoloaded functions in module autoloaded files.
+  4. Update Doom outside of Doom (e.g. with git)
+
+It will ensure that unneeded packages are removed, all needed packages are
+installed, autoloads files are up-to-date and no byte-compiled files have gone
+stale."
+  (add-hook 'kill-emacs-hook #'doom--cli-abort-warning-h)
+  (print! (start "Synchronizing your config with Doom Emacs..."))
+  (unwind-protect
+      (print-group!
+       (delete-file doom-autoloads-file)
+       (when (and (not no-envvar-p)
+                  (file-exists-p doom-env-file))
+         (doom-cli-reload-env-file 'force))
+       (run-hooks 'doom-sync-pre-hook)
+       (doom-cli-packages-install)
+       (doom-cli-packages-build)
+       (when update-p
+         (doom-cli-packages-update))
+       (doom-cli-packages-purge purge-p 'builds-p purge-p purge-p)
+       (run-hooks 'doom-sync-post-hook)
+       (when (doom-autoloads-reload)
+         (print! (info "Restart Emacs or use 'M-x doom/reload' for changes to take effect")))
+       t)
+    (remove-hook 'kill-emacs-hook #'doom--cli-abort-warning-h)))
+
+
+;;
+;;; DEPRECATED Commands
+
+(defcli! (refresh re) ()
+  "Deprecated for 'doom sync'"
+  :hidden t
+  (user-error "'doom refresh' has been replaced with 'doom sync'. Use that instead"))
+
+
+;;
+;;; Helpers
+
+(defun doom--cli-abort-warning-h ()
+  (terpri)
+  (print! (warn "Script was abruptly aborted! Run 'doom sync' to repair inconsistencies")))

core/cli/upgrade.el

@@ -24,7 +24,7 @@ following shell commands:
       ;; Reload Doom's CLI & libraries, in case there were any upstream changes.
       ;; Major changes will still break, however
       (print! (info "Reloading Doom Emacs"))
-      (doom-cli-execute-after "doom" "upgrade" "-p" (if force-p "-f")))
+      (throw 'exit (list "doom" "upgrade" "-p" (if force-p "-f"))))
 
      ((print! "Doom is up-to-date!")))))
 

core/core-cli.el

@@ -1,17 +1,10 @@
-;;; -*- lexical-binding: t; no-byte-compile: t; -*-
+;;; core/core-cli.el --- -*- lexical-binding: t; no-byte-compile: t; -*-
-
-(require 'seq)
 
 (load! "autoload/process")
 (load! "autoload/plist")
 (load! "autoload/files")
 (load! "autoload/output")
-
+(require 'seq)
-;; Create all our core directories to quell file errors
-(mapc (doom-rpartial #'make-directory 'parents)
-      (list doom-local-dir
-            doom-etc-dir
-            doom-cache-dir))
 
 ;; Ensure straight and the bare minimum is ready to go
 (require 'core-modules)
@@ -40,10 +33,26 @@ These are loaded when a Doom's CLI starts up. There users and modules can define
 additional CLI commands, or reconfigure existing ones to better suit their
 purpose.")
 
+(defvar doom-cli-log-file (concat doom-local-dir "doom.log")
+  "File to write the extended output to.")
+
+(defvar doom-cli-log-error-file (concat doom-local-dir "doom.error.log")
+  "File to write the last backtrace to.")
+
 (defvar doom--cli-commands (make-hash-table :test 'equal))
 (defvar doom--cli-groups (make-hash-table :test 'equal))
 (defvar doom--cli-group nil)
 
+(define-error 'doom-cli-error "There was an unexpected error" 'doom-error)
+(define-error 'doom-cli-command-not-found-error "Could not find that command" 'doom-cli-error)
+(define-error 'doom-cli-wrong-number-of-arguments-error "Wrong number of CLI arguments" 'doom-cli-error)
+(define-error 'doom-cli-unrecognized-option-error "Not a recognized option" 'doom-cli-error)
+(define-error 'doom-cli-deprecated-error "Command is deprecated" 'doom-cli-error)
+
+
+;;
+;;; CLI library
+
 (cl-defstruct
     (doom-cli
      (:constructor nil)
@@ -155,11 +164,7 @@ purpose.")
                          (command))
                    doom--cli-commands)))))
 
-(defun doom-cli-internal-p (cli)
+(defun doom-cli-execute (command &rest args)
-  "Return non-nil if CLI is an internal (non-public) command."
-  (string-prefix-p ":" (doom-cli-name cli)))
-
-(defun doom-cli-execute (command &optional args)
   "Execute COMMAND (string) with ARGS (list of strings).
 
 Executes a cli defined with `defcli!' with the name or alias specified by
@@ -169,45 +174,6 @@ COMMAND, and passes ARGS to it."
                (doom--cli-process cli (remq nil args)))
     (user-error "Couldn't find any %S command" command)))
 
-(defun doom-cli--execute-after (lines)
-  (let ((post-script (concat doom-local-dir ".doom.sh"))
-        (coding-system-for-write 'utf-8-unix)
-        (coding-system-for-read  'utf-8-unix))
-    (with-temp-file post-script
-      (insert "#!/usr/bin/env sh\n"
-              "_postscript() {\n"
-              "rm -f " (shell-quote-argument post-script) "\n"
-              (if (stringp lines)
-                  lines
-                (string-join
-                 (if (listp (car-safe lines))
-                     (cl-loop for line in (doom-enlist lines)
-                              collect (mapconcat #'shell-quote-argument (remq nil line) " "))
-                   (list (mapconcat #'shell-quote-argument (remq nil lines) " ")))
-                 "\n"))
-              "\n}\n"
-              (save-match-data
-                (cl-loop for env in process-environment
-                         if (string-match "^\\([a-zA-Z0-9_]+\\)=\\(.+\\)$" env)
-                         concat (format "%s=%s \\\n"
-                                        (match-string 1 env)
-                                        (shell-quote-argument (match-string 2 env)))))
-              (format "PATH=\"%s%s$PATH\" \\\n" (concat doom-emacs-dir "bin/") path-separator)
-              "_postscript $@\n"))
-    (set-file-modes post-script #o700)))
-
-(defun doom-cli-execute-lines-after (&rest lines)
-  "TODO"
-  (doom-cli--execute-after (string-join lines "\n")))
-
-(defun doom-cli-execute-after (&rest args)
-  "Execute shell command ARGS after this CLI session quits.
-
-This is particularly useful when the capabilities of Emacs' batch terminal are
-insufficient (like opening an instance of Emacs, or reloading Doom after a 'doom
-upgrade')."
-  (doom-cli--execute-after args))
-
 (defmacro defcli! (name speclist &optional docstring &rest body)
   "Defines a CLI command.
 
@@ -268,6 +234,63 @@ BODY will be run when this dispatcher is called."
      ,@body))
 
 
+;;
+;;; Debugger
+
+(defun doom-cli--debugger (&rest args)
+  (cl-incf num-nonmacro-input-events)
+  (cl-destructuring-bind (error data backtrace)
+      (list (caadr args)
+            (cdadr args)
+            (doom-cli--backtrace))
+    (print! (error "There was an unexpected error"))
+    (print-group!
+     (print! "%s %s" (bold "Message:") (get error 'error-message))
+     (print! "%s %s" (bold "Data:") (cons error data))
+     (when (and (bound-and-true-p straight-process-buffer)
+                (string-match-p (regexp-quote straight-process-buffer)
+                                (get error 'error-message)))
+       (print! (bold "Straight output:"))
+       (let ((output (straight--process-get-output)))
+         (appendq! data (list (cons "STRAIGHT" output)))
+         (print-group! (print! "%s" output))))
+     (when backtrace
+       (print! (bold "Backtrace:"))
+       (print-group!
+        (dolist (frame (seq-take backtrace 10))
+          (print! "%0.76s" frame)))
+       (with-temp-file doom-cli-log-error-file
+         (insert "# -*- lisp-interaction -*-\n")
+         (insert "# vim: set ft=lisp:\n")
+         (let ((standard-output (current-buffer))
+               (print-quoted t)
+               (print-escape-newlines t)
+               (print-escape-control-characters t)
+               (print-level nil)
+               (print-circle nil))
+           (mapc #'print (cons (list error data) backtrace)))
+         (print! (warn "Extended backtrace logged to %s")
+                 (relpath doom-cli-log-error-file))))))
+  (throw 'exit 255))
+
+(defun doom-cli--backtrace ()
+  (let* ((n 0)
+         (frame (backtrace-frame n))
+         (frame-list nil)
+         (in-program-stack nil))
+    (while frame
+      (when in-program-stack
+        (push (cdr frame) frame-list))
+      (when (eq (elt frame 1) 'doom-cli--debugger)
+        (setq in-program-stack t))
+      (when (and (eq (elt frame 1) 'doom-cli-execute)
+                 (eq (elt frame 2) :doom))
+        (setq in-program-stack nil))
+      (setq n (1+ n)
+            frame (backtrace-frame n)))
+    (reverse frame-list)))
+
+
 ;;
 ;;; straight.el hacks
 
@@ -387,52 +410,100 @@ everywhere we use it (and internally)."
           interactive)))
 
 
+;;
+;;; Entry point
+
+(defcli! :doom
+    ((help-p        ["-h" "--help"]  "Same as help command")
+     (auto-accept-p ["-y" "--yes"]   "Auto-accept all confirmation prompts")
+     (debug-p       ["-d" "--debug"] "Enables on verbose output")
+     (doomdir       ["--doomdir"  dir] "Use the private module at DIR (e.g. ~/.doom.d)")
+     (localdir      ["--localdir" dir] "Use DIR as your local storage directory")
+     &optional command
+     &rest args)
+  "A command line interface for managing Doom Emacs.
+
+Includes package management, diagnostics, unit tests, and byte-compilation.
+
+This tool also makes it trivial to launch Emacs out of a different folder or
+with a different private module."
+  (condition-case e
+      (with-output-to! doom-cli-log-file
+        (catch 'exit
+          (when (and (not (getenv "__DOOMRESTART"))
+                     (or doomdir
+                         localdir
+                         debug-p
+                         auto-accept-p))
+            (when doomdir
+              (setenv "DOOMDIR" (file-name-as-directory doomdir))
+              (print! (info "DOOMDIR=%s") localdir))
+            (when localdir
+              (setenv "DOOMLOCALDIR" (file-name-as-directory localdir))
+              (print! (info "DOOMLOCALDIR=%s") localdir))
+            (when debug-p
+              (setenv "DEBUG" "1")
+              (print! (info "DEBUG=1")))
+            (when auto-accept-p
+              (setenv "YES" auto-accept-p)
+              (print! (info "Confirmations auto-accept enabled")))
+            (setenv "__DOOMRESTART" "1")
+            (throw 'exit :restart))
+          (when help-p
+            (when command
+              (push command args))
+            (setq command "help"))
+          (if (null command)
+              (doom-cli-execute "help")
+            (let ((start-time (current-time)))
+              (run-hooks 'doom-cli-pre-hook)
+              (when (apply #'doom-cli-execute command args)
+                (run-hooks 'doom-cli-post-hook)
+                (print! (success "Finished in %.4fs")
+                        (float-time (time-subtract (current-time) start-time))))))))
+    ;; TODO Not implemented yet
+    (doom-cli-command-not-found-error
+     (print! (error "Command 'doom %s' not recognized") (string-join (cdr e) " "))
+     (print! "\nDid you mean one of these commands?")
+     (doom-cli-execute "help" "--similar" (string-join (cdr e) " "))
+     2)
+    ;; TODO Not implemented yet
+    (doom-cli-wrong-number-of-arguments-error
+     (cl-destructuring-bind (route opt arg n d) (cdr e)
+       (print! (error "doom %s: %S requires %d arguments, but %d given\n")
+               (mapconcat #'symbol-name route " ") arg n d)
+       (print-group!
+        (apply #'doom-cli-execute "help" (mapcar #'symbol-name route))))
+     3)
+    ;; TODO Not implemented yet
+    (doom-cli-unrecognized-option-error
+     (let ((option (cadr e)))
+       (print! (error "Unrecognized option: %S") option)
+       (when (string-match "^--[^=]+=\\(.+\\)$" option)
+         (print! "The %S syntax isn't supported. Use '%s %s' instead."
+                 option (car (split-string option "="))
+                 (match-string 1 option))))
+     4)
+    ;; TODO Not implemented yet
+    (doom-cli-deprecated-error
+     (cl-destructuring-bind (route . commands) (cdr e)
+       (print! (warn "The 'doom %s' command was removed and replaced with:\n")
+               (mapconcat #'symbol-name route " "))
+       (print-group!
+        (dolist (command commands)
+          (print! (info "%s") command))))
+     5)
+    (user-error
+     (print! (warn "%s") (cadr e))
+     1)))
+
+
 ;;
 ;;; CLI Commands
 
 (load! "cli/help")
 (load! "cli/install")
-
+(load! "cli/sync")
-(defcli! (refresh re) ()
-  "Deprecated for 'doom sync'"
-  :hidden t
-  (user-error "'doom refresh' has been replaced with 'doom sync'. Use that instead"))
-
-(defcli! (sync s)
-    ((inhibit-envvar-p ["-e"] "Don't regenerate the envvar file")
-     (inhibit-elc-p    ["-c"] "Don't recompile config")
-     (update-p         ["-u"] "Update installed packages after syncing")
-     (prune-p          ["-p" "--prune"] "Purge orphaned package repos & regraft them"))
-  "Synchronize your config with Doom Emacs.
-
-This is the equivalent of running autoremove, install, autoloads, then
-recompile. Run this whenever you:
-
-  1. Modify your `doom!' block,
-  2. Add or remove `package!' blocks to your config,
-  3. Add or remove autoloaded functions in module autoloaded files.
-  4. Update Doom outside of Doom (e.g. with git)
-
-It will ensure that unneeded packages are removed, all needed packages are
-installed, autoloads files are up-to-date and no byte-compiled files have gone
-stale."
-  (print! (start "Synchronizing your config with Doom Emacs..."))
-  (print-group!
-   (delete-file doom-autoload-file)
-   (when (and (not inhibit-envvar-p)
-              (file-exists-p doom-env-file))
-     (doom-cli-reload-env-file 'force))
-   (run-hooks 'doom-sync-pre-hook)
-   (doom-cli-packages-install)
-   (doom-cli-packages-build)
-   (when update-p
-     (doom-cli-packages-update))
-   (doom-cli-packages-purge prune-p 'builds-p prune-p prune-p)
-   (run-hooks 'doom-sync-post-hook)
-   (when (doom-autoloads-reload)
-     (print! (info "Restart Emacs or use 'M-x doom/reload' for changes to take effect")))
-   t))
-
 (load! "cli/env")
 (load! "cli/upgrade")
 (load! "cli/packages")
@@ -448,12 +519,10 @@ stale."
   ;; (load! "cli/test")
   )
 
-
 (defcligroup! "Compilation"
   "For compiling Doom and your config"
   (load! "cli/byte-compile"))
 
-
 (defcligroup! "Utilities"
   "Conveniences for interacting with Doom externally"
   (defcli! run (&rest args)
@@ -467,8 +536,27 @@ All arguments are passed on to Emacs.
 WARNING: this command exists for convenience and testing. Doom will suffer
 additional overhead by being started this way. For the best performance, it is
 best to run Doom out of ~/.emacs.d and ~/.doom.d."
-    (apply #'doom-cli-execute-after invocation-name args)
+    (throw 'exit (cons invocation-name args))))
-    nil))
+
+
+;;
+;;; Bootstrap
+
+(doom-log "Initializing Doom CLI")
+
+;; Clean slate for the next invocation
+(delete-file doom-cli-log-file)
+(delete-file doom-cli-log-error-file)
+
+;; Create all our core directories to quell file errors
+(mapc (doom-rpartial #'make-directory 'parents)
+      (list doom-local-dir
+            doom-etc-dir
+            doom-cache-dir))
+
+(load! doom-module-init-file doom-private-dir t)
+;; (maphash (doom-module-loader doom-cli-file) (doom-current-modules))
+(load! doom-cli-file doom-private-dir t)
 
 (provide 'core-cli)
 ;;; core-cli.el ends here

core/core-modules.el

@@ -443,7 +443,7 @@ otherwise, MODULES is a multiple-property list (a plist where each key can have
 multiple, linear values).
 
 The bootstrap process involves making sure the essential directories exist, core
-packages are installed, `doom-autoload-file' is loaded, `doom-packages-file'
+packages are installed, `doom-autoloads-file' is loaded, `doom-packages-file'
 cache exists (and is loaded) and, finally, loads your private init.el (which
 should contain your `doom!' block).
 

core/core.el

@@ -127,7 +127,7 @@ Use this for files that change often, like cache files. Must end with a slash.")
 Defaults to ~/.config/doom, ~/.doom.d or the value of the DOOMDIR envvar;
 whichever is found first. Must end in a slash.")
 
-(defconst doom-autoload-file (concat doom-local-dir "autoloads.el")
+(defconst doom-autoloads-file (concat doom-local-dir "autoloads.el")
   "Where `doom-reload-core-autoloads' stores its core autoloads.
 
 This file is responsible for informing Emacs where to find all of Doom's
@@ -492,7 +492,7 @@ If RETURN-P, return the message as a string instead of displaying it."
   "Bootstrap Doom, if it hasn't already (or if FORCE-P is non-nil).
 
 The bootstrap process ensures that everything Doom needs to run is set up;
-essential directories exist, core packages are installed, `doom-autoload-file'
+essential directories exist, core packages are installed, `doom-autoloads-file'
 is loaded (failing if it isn't), that all the needed hooks are in place, and
 that `core-packages' will load when `package' or `straight' is used.
 
@@ -524,7 +524,7 @@ to least)."
                   load-path doom--initial-load-path
                   process-environment doom--initial-process-environment)
 
-    ;; Doom caches a lot of information in `doom-autoload-file'. Module and
+    ;; Doom caches a lot of information in `doom-autoloads-file'. Module and
     ;; package autoloads, autodefs like `set-company-backend!', and variables
     ;; like `doom-modules', `doom-disabled-packages', `load-path',
     ;; `auto-mode-alist', and `Info-directory-list'. etc. Compiling them into
@@ -533,15 +533,15 @@ to least)."
         ;; Avoid `file-name-sans-extension' for premature optimization reasons.
         ;; `string-remove-suffix' is cheaper because it performs no file sanity
         ;; checks; just plain ol' string manipulation.
-        (load (string-remove-suffix ".el" doom-autoload-file)
+        (load (string-remove-suffix ".el" doom-autoloads-file)
               nil 'nomessage)
       (file-missing
        ;; If the autoloads file fails to load then the user forgot to sync, or
        ;; aborted a doom command midway!
-       (if (equal (nth 3 e) doom-autoload-file)
+       (if (equal (nth 3 e) doom-autoloads-file)
            (signal 'doom-error
                    (list "Doom is in an incomplete state"
-                         "run 'bin/doom sync' on the command line to repair it"))
+                         "run 'doom sync' on the command line to repair it"))
          ;; Otherwise, something inside the autoloads file is triggering this
          ;; error; forward it!
          (apply #'doom-autoload-error e))))

core/test/test-core.el

@@ -28,7 +28,7 @@
           (spy-on 'doom-initialize-packages :and-return-value t))
 
         (it "initializes packages if core autoload file doesn't exist"
-          (let ((doom-autoload-file "doesnotexist"))
+          (let ((doom-autoloads-file "doesnotexist"))
             (expect (doom-initialize nil 'noerror))
           (expect 'doom-initialize-packages :to-have-been-called))
 
@@ -51,12 +51,12 @@
         (it "loads autoloads files"
           (ignore-errors (doom-initialize nil 'noerror))
           (expect 'doom-load-autoloads-file
-                  :to-have-been-called-with doom-autoload-file)
+                  :to-have-been-called-with doom-autoloads-file)
           (expect 'doom-load-autoloads-file
                   :to-have-been-called-with doom-package-autoload-file))
 
         (it "throws doom-autoload-error when autoload files don't exist"
-          (let ((doom-autoload-file "doesnotexist")
+          (let ((doom-autoloads-file "doesnotexist")
                 (doom-package-autoload-file "doesnotexist"))
             (expect (doom-initialize) :to-throw 'doom-autoload-error)))))
 
@@ -72,26 +72,26 @@
         (expect 'require :to-have-been-called-with 'core-editor))))
 
   (describe "doom-load-autoloads-file"
-    :var (doom-autoload-file doom-alt-autoload-file result)
+    :var (doom-autoloads-file doom-alt-autoload-file result)
     (before-each
-      (setq doom-autoload-file (make-temp-file "doom-autoload" nil ".el"))
+      (setq doom-autoloads-file (make-temp-file "doom-autoload" nil ".el"))
-      (with-temp-file doom-autoload-file)
+      (with-temp-file doom-autoloads-file)
-      (byte-compile-file doom-autoload-file))
+      (byte-compile-file doom-autoloads-file))
     (after-each
-      (delete-file doom-autoload-file)
+      (delete-file doom-autoloads-file)
-      (delete-file (byte-compile-dest-file doom-autoload-file)))
+      (delete-file (byte-compile-dest-file doom-autoloads-file)))
 
     (it "loads the byte-compiled autoloads file if available"
-      (doom-load-autoloads-file doom-autoload-file)
+      (doom-load-autoloads-file doom-autoloads-file)
       (expect (caar load-history) :to-equal-file
-              (byte-compile-dest-file doom-autoload-file))
+              (byte-compile-dest-file doom-autoloads-file))
 
-      (delete-file (byte-compile-dest-file doom-autoload-file))
+      (delete-file (byte-compile-dest-file doom-autoloads-file))
-      (doom-load-autoloads-file doom-autoload-file)
+      (doom-load-autoloads-file doom-autoloads-file)
-      (expect (caar load-history) :to-equal-file doom-autoload-file))
+      (expect (caar load-history) :to-equal-file doom-autoloads-file))
 
     (it "returns non-nil if successful"
-      (expect (doom-load-autoloads-file doom-autoload-file)))
+      (expect (doom-load-autoloads-file doom-autoloads-file)))
 
     (it "returns nil on failure or error, non-fatally"
       (expect (doom-load-autoloads-file "/does/not/exist") :to-be nil)))