emenel/doomemacs
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

fix(cli): rewrite 'doom sync'; deprecate 'doom build'

Browse source 
This changes 'doom sync' to be smarter about responding to changed
package recipes/pins, changes in Emacs version, or instances where the user
has copied a config to a new system.

In all these cases, the user would formerly have to know about a
specific combination of 'doom sync -u' and 'doom build' to ensure Doom
is in a good state. With this change, 'doom sync' handles all these
cases.

Also, 'doom build' is now deprecated (and 'doom sync' now has a
--rebuild option to mimic its old behavior).

Also also, sometimes, a package may silently fail when cloned (which
used to result in an empty repo). Now, if this is detected, cloning will
be re-attempted up to 3 times before aborting with much more visible
error.

Note: these are stopgap solutions, until v3 is finished.
This commit is contained in:
Henrik Lissner 2024-03-24 17:38:40 -04:00
parent 1fa8d3a4b9
commit cff091982e
No known key found for this signature in database
GPG key ID: B60957CA074D39A3
6 changed files with 195 additions and 212 deletions
Download patch file Download diff file Expand all files Collapse all files

2
README.md
View file

@ -138,8 +138,6 @@ commands you should know about:
+ `doom env` to dump a snapshot of your shell environment to a file that Doom + `doom env` to dump a snapshot of your shell environment to a file that Doom
will load at startup. This allows Emacs to inherit your `PATH`, among other will load at startup. This allows Emacs to inherit your `PATH`, among other
things. things.
+ `doom build` to recompile all installed packages (use this if you up/downgrade
Emacs).
# Roadmap # Roadmap

1
bin/doom
View file

@ -280,7 +280,6 @@ SEE ALSO:
;; (load! "nuke" dir) ;; (load! "nuke" dir)
;; (load! "package" dir) ;; (load! "package" dir)
;; (load! "profile" dir) ;; (load! "profile" dir)
;; (defcli-obsolete! ((build b)) (sync "--rebuild") "v3.0.0")
) )
(defcli-group! "Diagnostics" (defcli-group! "Diagnostics"

7
docs/faq.org
View file

@ -402,14 +402,13 @@ This command is never needed for changes to =$DOOMDIR/config.el=.
** Copy or sync my config to another system? ** Copy or sync my config to another system?
*Short answer:* it is safe to sync =$DOOMDIR= across systems, but not *Short answer:* it is safe to sync =$DOOMDIR= across systems, but not
=$EMACSDIR=. Once moved, use ~$ doom sync && doom build~ to ensure everything is =$EMACSDIR=. Once moved, use ~$ doom sync~ to ensure everything is set up
set up correctly. correctly.
*Long answer:* packages can contain baked-in absolute paths and non-portable *Long answer:* packages can contain baked-in absolute paths and non-portable
byte-code. It is never a good idea to mirror it across multiple systems, unless byte-code. It is never a good idea to mirror it across multiple systems, unless
they are all the same (same OS, same version of Emacs, same paths). Most issues they are all the same (same OS, same version of Emacs, same paths). Most issues
should be solved by running ~$ doom sync && doom build~ on the other end, once should be solved by running ~$ doom sync~ on the other end, once moved.
moved.
** Start over, in case something went terribly wrong? ** Start over, in case something went terribly wrong?
Delete =$EMACSDIR/.local/straight= and run ~$ doom sync~. Delete =$EMACSDIR/.local/straight= and run ~$ doom sync~.

185
lisp/cli/packages.el
View file

@ -13,25 +13,8 @@
;; ;;
;;; Commands ;;; Commands
(defcli! (:before (build b purge p)) (&context context) (defcli-obsolete! ((build b)) (sync "--rebuild") "v3.0.0")
(require 'comp nil t)
(doom-initialize-core-packages))
;; DEPRECATED Replace with "doom sync --rebuild"
(defcli! ((build b))
((rebuild-p ("-r") "Only rebuild packages that need rebuilding")
(jobs ("-j" "--jobs" num) "How many CPUs to use for native compilation"))
"Byte-compiles & symlinks installed packages.
This ensures that all needed files are symlinked from their package repo and
their elisp files are byte-compiled. This is especially necessary if you upgrade
Emacs (as byte-code is generally not forward-compatible)."
:benchmark t
(when jobs
(setq native-comp-async-jobs-number (truncate jobs)))
(when (doom-packages-build (not rebuild-p))
(doom-profile-generate))
t)
;; TODO Rename to "doom gc" and move to its own file ;; TODO Rename to "doom gc" and move to its own file
(defcli! ((purge p)) (defcli! ((purge p))
@ -50,6 +33,8 @@ possible.
It is a good idea to occasionally run this doom purge -g to ensure your package It is a good idea to occasionally run this doom purge -g to ensure your package
list remains lean." list remains lean."
:benchmark t :benchmark t
(require 'comp nil t)
(doom-initialize-core-packages)
(straight-check-all) (straight-check-all)
(when (doom-packages-purge (when (doom-packages-purge
(not noelpa-p) (not noelpa-p)
@ -242,7 +227,6 @@ list remains lean."
(defun doom-packages--write-missing-eln-errors () (defun doom-packages--write-missing-eln-errors ()
"Write .error files for any expected .eln files that are missing." "Write .error files for any expected .eln files that are missing."
(when (featurep 'native-compile)
(cl-loop for file in doom-packages--eln-output-expected (cl-loop for file in doom-packages--eln-output-expected
for eln-name = (doom-packages--eln-file-name file) for eln-name = (doom-packages--eln-file-name file)
for eln-file = (doom-packages--eln-output-file eln-name) for eln-file = (doom-packages--eln-output-file eln-name)
@ -254,11 +238,10 @@ list remains lean."
do (make-directory error-dir 'parents) do (make-directory error-dir 'parents)
(write-region "" nil error-file) (write-region "" nil error-file)
(doom-log "Wrote %s" error-file)) (doom-log "Wrote %s" error-file))
(setq doom-packages--eln-output-expected nil))) (setq doom-packages--eln-output-expected nil))
(defun doom-packages--compile-site-files () (defun doom-packages--compile-site-files ()
"Queue async compilation for all non-doom Elisp files." "Queue async compilation for all non-doom Elisp files."
(when (featurep 'native-compile)
(cl-loop with paths = (cl-loop for path in load-path (cl-loop with paths = (cl-loop for path in load-path
unless (file-in-directory-p path doom-local-dir) unless (file-in-directory-p path doom-local-dir)
collect path) collect path)
@ -268,71 +251,16 @@ list remains lean."
(not (cl-some (fn! (string-match-p % file)) (not (cl-some (fn! (string-match-p % file))
native-comp-deferred-compilation-deny-list))) do native-comp-deferred-compilation-deny-list))) do
(doom-log "Compiling %s" file) (doom-log "Compiling %s" file)
(native-compile-async file)))) (native-compile-async file)))
(defun doom-packages-install () (defun doom-packages-ensure (&optional force-p)
"Installs missing packages. "Ensure packages are installed, built"
This function will install any primary package (i.e. a package with a `package!'
declaration) or dependency thereof that hasn't already been."
(doom-initialize-packages) (doom-initialize-packages)
(print! (start "Installing packages...")) (if (not (file-directory-p (straight--repos-dir)))
(let ((pinned (doom-package-pinned-list))) (print! (start "Installing all packages for the first time (this may take a while)..."))
(print-group! (if force-p
(add-hook 'native-comp-async-cu-done-functions #'doom-packages--native-compile-done-h) (print! (start "Rebuilding all packages (this may take a while)..."))
(if-let (built (print! (start "Ensuring packages are installed and built..."))))
(doom-packages--with-recipes (doom-package-recipe-list)
(recipe package type local-repo)
(let ((repo-dir (straight--repos-dir local-repo)))
(unless (file-directory-p repo-dir)
(doom-packages--cli-recipes-update))
(condition-case-unless-debug e
(let ((straight-use-package-pre-build-functions
(cons (lambda (pkg &rest _)
(when-let (commit (cdr (assoc pkg pinned)))
(print! (item "Checked out %s: %s") pkg commit)))
straight-use-package-pre-build-functions)))
;; HACK: Straight can sometimes fail to clone a repo,
;; leaving behind an empty directory which, in future
;; invocations, it will assume indicates a successful
;; clone (causing load errors later).
(let ((try 0))
(while (or (not (file-directory-p repo-dir))
(directory-empty-p repo-dir))
(if (= try 3)
(error "Failed to clone package")
(when (> try 0)
(print! "Failed to clone %S, trying again (attempt #%d)..." package (1+ try))))
(delete-file (file-name-concat (straight--modified-dir) package))
(delete-directory repo-dir t)
(delete-directory (straight--build-dir package) t)
(straight-use-package (intern package))
(cl-incf try)))
;; HACK: Line encoding issues can plague repos with
;; dirty worktree prompts when updating packages or
;; "Local variables entry is missing the suffix"
;; errors when installing them (see #2637), so have
;; git handle conversion by force.
(when (and doom--system-windows-p (stringp local-repo))
(let ((default-directory (straight--repos-dir local-repo)))
(when (file-in-directory-p default-directory straight-base-dir)
(straight--process-run "git" "config" "core.autocrlf" "true")))))
(error
(signal 'doom-package-error (list package e)))))))
(progn
(when (featurep 'native-compile)
(doom-packages--compile-site-files)
(doom-packages--wait-for-native-compile-jobs)
(doom-packages--write-missing-eln-errors))
(print! (success "\033[KInstalled %d packages") (length built)))
(print! (item "No packages need to be installed"))
nil))))
(defun doom-packages-build (&optional force-p)
"(Re)build all packages."
(doom-initialize-packages)
(print! (start "(Re)building %spackages...") (if force-p "all " ""))
(print-group! (print-group!
(let ((straight-check-for-modifications (let ((straight-check-for-modifications
(when (file-directory-p (straight--modified-dir)) (when (file-directory-p (straight--modified-dir))
@ -346,17 +274,17 @@ declaration) or dependency thereof that hasn't already been."
(straight--packages-to-rebuild (straight--packages-to-rebuild
(or (if force-p :all straight--packages-to-rebuild) (or (if force-p :all straight--packages-to-rebuild)
(make-hash-table :test #'equal))) (make-hash-table :test #'equal)))
(recipes (doom-package-recipe-list))) (recipes (doom-package-recipe-list))
(pinned (doom-package-pinned-list)))
(add-hook 'native-comp-async-cu-done-functions #'doom-packages--native-compile-done-h) (add-hook 'native-comp-async-cu-done-functions #'doom-packages--native-compile-done-h)
(unless force-p (straight--make-build-cache-available)
(straight--make-build-cache-available))
(if-let (built (if-let (built
(doom-packages--with-recipes recipes (package local-repo recipe) (doom-packages--with-recipes recipes (package local-repo recipe)
(let ((repo-dir (straight--repos-dir (or local-repo package)))
(build-dir (straight--build-dir package)))
(unless force-p (unless force-p
;; Ensure packages with outdated files/bytecode are rebuilt ;; Ensure packages with outdated files/bytecode are rebuilt
(let* ((build-dir (straight--build-dir package)) (let* ((build (if (plist-member recipe :build)
(repo-dir (straight--repos-dir local-repo))
(build (if (plist-member recipe :build)
(plist-get recipe :build) (plist-get recipe :build)
t)) t))
(want-byte-compile (want-byte-compile
@ -382,30 +310,66 @@ declaration) or dependency thereof that hasn't already been."
(push file doom-packages--eln-output-expected)) (push file doom-packages--eln-output-expected))
finally return outdated)) finally return outdated))
(puthash package t straight--packages-to-rebuild)))) (puthash package t straight--packages-to-rebuild))))
(straight-use-package (intern package)))) (unless (file-directory-p repo-dir)
(doom-packages--cli-recipes-update))
(condition-case-unless-debug e
(let ((straight-vc-git-post-clone-hook
(cons (lambda! (&key repo-dir commit)
(print-group!
(if-let (pin (cdr (assoc package pinned)))
(print! (item "Pinned to %s") pin)
(print! (item "Checked out %s") commit)))
;; HACK: Line encoding issues can plague
;; repos with dirty worktree prompts
;; when updating packages or "Local
;; variables entry is missing the
;; suffix" errors when installing them
;; (see #2637), so have git handle
;; conversion by force.
(when (and doom--system-windows-p (stringp repo-dir))
(let ((default-directory repo-dir))
(when (file-in-directory-p default-directory straight-base-dir)
(straight--process-run "git" "config" "core.autocrlf" "true")))))
straight-vc-git-post-clone-hook)))
(straight-use-package (intern package))
;; HACK: Straight can sometimes fail to clone a repo,
;; leaving behind an empty directory which, in future
;; invocations, it will assume indicates a successful
;; clone (causing load errors later).
(let ((try 0))
(while (or (not (file-directory-p repo-dir))
(directory-empty-p repo-dir))
(when (= try 3)
(error "Failed to clone package"))
(print! "Failed to clone %S, trying again (attempt #%d)..." package (1+ try))
(delete-directory repo-dir t)
(delete-directory build-dir t)
(straight-use-package (intern package))
(cl-incf try))))
(error
(signal 'doom-package-error (list package e)))))))
(progn (progn
(when (featurep 'native-compile) (when (featurep 'native-compile)
(doom-packages--compile-site-files) (doom-packages--compile-site-files)
(doom-packages--wait-for-native-compile-jobs) (doom-packages--wait-for-native-compile-jobs)
(doom-packages--write-missing-eln-errors)) (doom-packages--write-missing-eln-errors))
;; HACK Every time you save a file in a package that straight tracks, ;; HACK: Every time you save a file in a package that straight
;; it is recorded in ~/.emacs.d/.local/straight/modified/. ;; tracks, it is recorded in ~/.emacs.d/.local/straight/modified/.
;; Typically, straight will clean these up after rebuilding, but ;; Typically, straight will clean these up after rebuilding, but
;; Doom's use-case circumnavigates that, leaving these files ;; Doom's use-case circumnavigates that, leaving these files there
;; there and causing a rebuild of those packages each time `doom ;; and causing a rebuild of those packages each time `doom sync'
;; sync' or similar is run, so we clean it up ourselves: ;; or similar is run, so we clean it up ourselves:
(delete-directory (straight--modified-dir) 'recursive) (delete-directory (straight--modified-dir) 'recursive)
(print! (success "\033[KRebuilt %d package(s)") (length built))) (print! (success "\033[KBuilt %d package(s)") (length built)))
(print! (item "No packages need rebuilding")) (print! (item "No packages need attention"))
nil)))) nil))))
(defun doom-packages-update () (defun doom-packages-update (&optional pinned-only-p)
"Updates packages." "Updates packages."
(doom-initialize-packages) (doom-initialize-packages)
(doom-packages--barf-if-incomplete) (doom-packages--barf-if-incomplete)
(doom-packages--cli-recipes-update)
(let* ((repo-dir (straight--repos-dir)) (let* ((repo-dir (straight--repos-dir))
(pinned (doom-package-pinned-list)) (pinned (doom-package-pinned-list))
(recipes (doom-package-recipe-list)) (recipes (doom-package-recipe-list))
@ -413,9 +377,10 @@ declaration) or dependency thereof that hasn't already been."
(repos-to-rebuild (make-hash-table :test 'equal)) (repos-to-rebuild (make-hash-table :test 'equal))
(total (length recipes)) (total (length recipes))
(esc (unless init-file-debug "\033[1A")) (esc (unless init-file-debug "\033[1A"))
(i 0) (i 0))
errors) (if pinned-only-p
(print! (start "Updating packages (this may take a while)...")) (print! (start "Updating pinned packages..."))
(print! (start "Updating all packages (this may take a while)...")))
(doom-packages--with-recipes recipes (recipe package type local-repo) (doom-packages--with-recipes recipes (recipe package type local-repo)
(cl-incf i) (cl-incf i)
(print-group! (print-group!
@ -428,11 +393,13 @@ declaration) or dependency thereof that hasn't already been."
(cl-return)) (cl-return))
(let ((default-directory (straight--repos-dir local-repo))) (let ((default-directory (straight--repos-dir local-repo)))
(unless (file-in-directory-p default-directory repo-dir) (unless (file-in-directory-p default-directory repo-dir)
(print! (warn "(%d/%d) Skipping %s because it is local") i total package) (print! (warn "(%d/%d) Skipping %s because it is out-of-tree...") i total package)
(cl-return)) (cl-return))
(when (eq type 'git) (when (eq type 'git)
(unless (file-exists-p ".git") (unless (file-exists-p ".git")
(error "%S is not a valid repository" package))) (error "%S is not a valid repository" package)))
(when (and pinned-only-p (not (assoc local-repo pinned)))
(cl-return))
(condition-case-unless-debug e (condition-case-unless-debug e
(let ((ref (straight-vc-get-commit type local-repo)) (let ((ref (straight-vc-get-commit type local-repo))
(target-ref (target-ref
@ -446,13 +413,6 @@ declaration) or dependency thereof that hasn't already been."
(doom-packages--straight-with (straight-vc-fetch-from-remote recipe) (doom-packages--straight-with (straight-vc-fetch-from-remote recipe)
(when .it (when .it
(straight-merge-package package) (straight-merge-package package)
;; (condition-case e
;; (straight-merge-package package)
;; (wrong-type-argument
;; (if (not (equal (cdr e) '(arrayp nil)))
;; (signal (car e) (cdr e))
;; (delete-directory (straight--build-dir local-repo) t)
;; (straight-use-package (intern package)))))
(setq target-ref (straight-vc-get-commit type local-repo)) (setq target-ref (straight-vc-get-commit type local-repo))
(setq output (doom-packages--commit-log-between ref target-ref) (setq output (doom-packages--commit-log-between ref target-ref)
commits (length (split-string output "\n" t))) commits (length (split-string output "\n" t)))
@ -536,13 +496,14 @@ declaration) or dependency thereof that hasn't already been."
(princ "\033[K") (princ "\033[K")
(if (hash-table-empty-p packages-to-rebuild) (if (hash-table-empty-p packages-to-rebuild)
(ignore (print! (success "All %d packages are up-to-date") total)) (ignore (print! (success "All %d packages are up-to-date") total))
(doom-packages--cli-recipes-update)
(straight--transaction-finalize) (straight--transaction-finalize)
(let ((default-directory (straight--build-dir))) (let ((default-directory (straight--build-dir)))
(mapc (doom-rpartial #'delete-directory 'recursive) (mapc (doom-rpartial #'delete-directory 'recursive)
(hash-table-keys packages-to-rebuild))) (hash-table-keys packages-to-rebuild)))
(print! (success "Updated %d package(s)") (print! (success "Updated %d package(s)")
(hash-table-count packages-to-rebuild)) (hash-table-count packages-to-rebuild))
(doom-packages-build) (doom-packages-ensure)
t)))) t))))

58
lisp/cli/sync.el
View file

@ -14,18 +14,21 @@
(defvar doom-before-sync-hook () (defvar doom-before-sync-hook ()
"Hooks run before 'doom sync' synchronizes the user's config with Doom.") "Hooks run before 'doom sync' synchronizes the user's config with Doom.")
(defvar doom-cli-sync-info-file (file-name-concat doom-profile-data-dir "sync"))
;; ;;
;;; Commands ;;; Commands
(defcli-alias! (:before (sync s)) (:before build))
(defcli! ((sync s)) (defcli! ((sync s))
((noenvvar? ("-e") "Don't regenerate the envvar file") ((noenvvar? ("-e") "Don't regenerate the envvar file")
(noelc? ("-c") "Don't recompile config") (update? ("-u") "Update all installed packages after syncing")
(update? ("-u") "Update installed packages after syncing") (noupdate? ("-U") "Don't update any packages")
(purge? ("-p") "Purge orphaned package repos & regraft them") (purge? ("-p") "Purge orphaned package repos & regraft them")
(jobs ("-j" "--jobs" num) "How many CPUs to use for native compilation")) (jobs ("-j" "--jobs" num) "How many threads to use for native compilation")
(rebuild? ("-b" "--rebuild") "Rebuild, compile, & symlink installed packages")
(auto? ("-B") "Rebuild packages, but only if necessary")
&context context)
"Synchronize your config with Doom Emacs. "Synchronize your config with Doom Emacs.
This is the equivalent of running autoremove, install, autoloads, then This is the equivalent of running autoremove, install, autoloads, then
@ -33,8 +36,10 @@ recompile. Run this whenever you:
1. Modify your `doom!' block, 1. Modify your `doom!' block,
2. Add or remove `package!' blocks to your config, 2. Add or remove `package!' blocks to your config,
3. Add or remove autoloaded functions in module autoloaded files. 3. Add or remove autoloaded functions in module autoloaded files,
4. Update Doom outside of Doom (e.g. with git) 4. Update Doom outside of Doom (e.g. with git),
5. Move your Doom config (either $EMACSDIR or $DOOMDIR) to a new location.
6. When you up (or down) grade Emacs itself.
It will ensure that unneeded packages are removed, all needed packages are It will ensure that unneeded packages are removed, all needed packages are
installed, autoloads files are up-to-date and no byte-compiled files have gone installed, autoloads files are up-to-date and no byte-compiled files have gone
@ -47,25 +52,46 @@ OPTIONS:
:benchmark t :benchmark t
(when (doom-profiles-bootloadable-p) (when (doom-profiles-bootloadable-p)
(call! '(profiles sync "--reload"))) (call! '(profiles sync "--reload")))
(run-hooks 'doom-before-sync-hook) (when (doom-cli-context-suppress-prompts-p context)
(add-hook 'kill-emacs-hook #'doom-sync--abort-warning-h) (setq auto? t))
(when jobs (when jobs
(setq native-comp-async-jobs-number (truncate jobs))) (setq native-comp-async-jobs-number (truncate jobs)))
(print! (start "Synchronizing %S profile..." ) (run-hooks 'doom-before-sync-hook)
(or (car doom-profile) "default")) (add-hook 'kill-emacs-hook #'doom-sync--abort-warning-h)
(print! (item "Using Emacs %s @ %s") emacs-version (path invocation-directory invocation-name))
(print! (start "Synchronizing %S profile..." ) (or (car doom-profile) "default"))
(unwind-protect (unwind-protect
(print-group! (print-group!
;; If the user has up/downgraded Emacs since last sync, or copied their
;; config to a different system, then their packages need to be
;; recompiled. This is necessary because Emacs byte-code is not
;; necessarily back/forward compatible across major versions, and many
;; packages bake in hardcoded data at compile-time.
(pcase-let ((`(,old-version . ,old-host) (doom-file-read doom-cli-sync-info-file :by 'read :noerror t))
(to-rebuild nil))
(when (and old-version (not (equal old-version emacs-version)))
(print! (warn "Emacs version has changed since last sync (from %s to %s)") old-version emacs-version)
(setq to-rebuild t))
(when (and old-host (not (equal old-host (system-name))))
(print! (warn "Your system has changed since last sync"))
(setq to-rebuild t))
(when (and to-rebuild (not auto?))
(or (y-or-n-p
(format! " %s" "Your installed packages will need to be recompiled. Do so now?"))
(exit! 0))
(setq rebuild? t)))
(when (and (not noenvvar?) (when (and (not noenvvar?)
(file-exists-p doom-env-file)) (file-exists-p
(file-name-concat doom-profile-dir
doom-profile-env-file-name)))
(call! '(env))) (call! '(env)))
(doom-packages-install) (doom-packages-ensure rebuild?)
(doom-packages-build) (unless noupdate? (doom-packages-update (not update?)))
(when update?
(doom-packages-update))
(doom-packages-purge purge? purge? purge? purge? purge?) (doom-packages-purge purge? purge? purge? purge? purge?)
(when (doom-profile-generate) (when (doom-profile-generate)
(print! (item "Restart Emacs or use 'M-x doom/reload' for changes to take effect")) (print! (item "Restart Emacs or use 'M-x doom/reload' for changes to take effect"))
(run-hooks 'doom-after-sync-hook)) (run-hooks 'doom-after-sync-hook))
(with-temp-file doom-cli-sync-info-file (prin1 (cons emacs-version (system-name)) (current-buffer)))
t) t)
(remove-hook 'kill-emacs-hook #'doom-sync--abort-warning-h))) (remove-hook 'kill-emacs-hook #'doom-sync--abort-warning-h)))

2
lisp/cli/upgrade.el
View file

@ -32,7 +32,7 @@ following shell commands:
doom clean doom clean
doom sync -u" doom sync -u"
(let* ((force? (doom-cli-context-suppress-prompts-p context)) (let* ((force? (doom-cli-context-suppress-prompts-p context))
(sync-cmd (append '("sync" "-u") (if jobs `("-j" ,num))))) (sync-cmd (append '("sync" "-u" "-B") (if jobs `("-j" ,num)))))
(cond (cond
(packages? (packages?
;; HACK It's messy to use straight to upgrade straight, due to the ;; HACK It's messy to use straight to upgrade straight, due to the