From 89c464ddf7eaf4e65b052d41afc3913ae3fcf2fe Mon Sep 17 00:00:00 2001 From: Andrew Whatson Date: Thu, 8 Oct 2020 09:03:14 +1000 Subject: [PATCH 1/4] Bump straight.el raxod502/straight.el@0c7c7571 -> raxod502/straight.el@728ea18e --- core/packages.el | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/packages.el b/core/packages.el index 45972127a..d46c9e89e 100644 --- a/core/packages.el +++ b/core/packages.el @@ -18,7 +18,7 @@ :local-repo "straight.el" :files ("straight*.el") :no-build t) - :pin "0c7c7571349b628d87acde474a754f05e86ca876") + :pin "728ea18ea590fcd8fb48f5bed30e135942d97221") ;; core-modules.el (package! use-package From 30140021debe63c9b50b32bc72f3771644e1a8cb Mon Sep 17 00:00:00 2001 From: Andrew Whatson Date: Thu, 8 Oct 2020 09:20:36 +1000 Subject: [PATCH 2/4] Export comp-deferred-compilation-black-list via autoloads The latest straight.el adds `:no-native-compile` packages to the compilation blacklist. We export the build-time blacklist via autoloads so that it works as expected to prevent native compilation at runtime. --- core/cli/autoloads.el | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/core/cli/autoloads.el b/core/cli/autoloads.el index c471b01cb..50a94b779 100644 --- a/core/cli/autoloads.el +++ b/core/cli/autoloads.el @@ -10,6 +10,7 @@ one wants that.") (defvar doom-autoloads-cached-vars '(doom-modules doom-disabled-packages + comp-deferred-compilation-black-list load-path auto-mode-alist interpreter-mode-alist @@ -42,8 +43,9 @@ one wants that.") (signal 'doom-error (list "The installed version of Doom has changed since last 'doom sync' ran" "Run 'doom sync' to bring Doom up to speed")))) - (mapcar (lambda (var) `(set ',var ',(symbol-value var))) - doom-autoloads-cached-vars) + (cl-loop for var in doom-autoloads-cached-vars + when (boundp var) + collect `(set ',var ',(symbol-value var))) (doom-autoloads--scan (append (cl-loop for dir in (append (list doom-core-dir) From 1d0f1575aa64a6207912d2ba7fcdbdceed7b153e Mon Sep 17 00:00:00 2001 From: Andrew Whatson Date: Thu, 8 Oct 2020 09:04:55 +1000 Subject: [PATCH 3/4] Remove `comp-eln-load-path` injection hack This was fixed upstream, so comp-eln-load-path is now passed to async compilation workers. --- core/core.el | 7 ------- 1 file changed, 7 deletions(-) diff --git a/core/core.el b/core/core.el index 378ba8a28..a0b6e636d 100644 --- a/core/core.el +++ b/core/core.el @@ -284,13 +284,6 @@ config.el instead." (add-to-list 'comp-eln-load-path (concat doom-cache-dir "eln/"))) (after! comp - ;; HACK `comp-eln-load-path' isn't fully respected yet, because native - ;; compilation occurs in another emacs process that isn't seeded with our - ;; value for `comp-eln-load-path', so we inject it ourselves: - (setq comp-async-env-modifier-form - `(progn - ,comp-async-env-modifier-form - (setq comp-eln-load-path ',(bound-and-true-p comp-eln-load-path)))) ;; HACK Disable native-compilation for some troublesome packages (add-to-list 'comp-deferred-compilation-black-list "/evil-collection-vterm\\.el\\'")) From 2e07a85590bfcccfcabc3f5e720553961c6def81 Mon Sep 17 00:00:00 2001 From: Andrew Whatson Date: Wed, 7 Oct 2020 22:23:51 +1000 Subject: [PATCH 4/4] Native-compile everything on load-path Add an extra pass to `doom build` to queue native compilation of all packages on `load-path`. This ensures that all core and site Elisp packages are actually native-compiled, even on "fast boot" builds. Add `.local/autolads.el` to the blacklist as native-compiling this file succeeds, but prevents emacs from starting up. Revise the evil-collection-vterm blacklist entry to use a more efficient regex. Fix a bug where an updated .error file wasn't always written, causing spurious rebuilds. --- core/cli/packages.el | 110 ++++++++++++++++++++++++++++++------------- core/core.el | 4 +- 2 files changed, 79 insertions(+), 35 deletions(-) diff --git a/core/cli/packages.el b/core/cli/packages.el index 375ff19b7..3abbacf5a 100644 --- a/core/cli/packages.el +++ b/core/cli/packages.el @@ -120,9 +120,34 @@ list remains lean." (setq straight--recipe-lookup-cache (make-hash-table :test #'eq) doom--cli-updated-recipes t))) -(defvar doom--expected-eln-files nil) +(defvar doom--eln-output-expected nil) + +(defvar doom--eln-output-path (car comp-eln-load-path)) + +(defun doom--eln-file-name (file) + "Return the short .eln file name corresponding to `file'." + (concat comp-native-version-dir "/" + (file-name-nondirectory + (comp-el-to-eln-filename file)))) + +(defun doom--eln-output-file (eln-name) + "Return the expected .eln file corresponding to `eln-name'." + (concat doom--eln-output-path eln-name)) + +(defun doom--eln-error-file (eln-name) + "Return the expected .error file corresponding to `eln-name'." + (concat doom--eln-output-path eln-name ".error")) + +(defun doom--find-eln-file (eln-name) + "Find `eln-name' on the `comp-eln-load-path'." + (cl-some (lambda (eln-path) + (let ((file (concat eln-path eln-name))) + (when (file-exists-p file) + file))) + comp-eln-load-path)) (defun doom--elc-file-outdated-p (file) + "Check whether the corresponding .elc for `file' is outdated." (let ((elc-file (byte-compile-dest-file file))) ;; NOTE Ignore missing elc files, they could be missing due to ;; `no-byte-compile'. Rebuilding unnecessarily is expensive. @@ -131,17 +156,12 @@ list remains lean." (doom-log "%s is newer than %s" file elc-file) t))) -;; DEPRECATED Remove later -(defun doom--comp-output-filename (file) - (if (fboundp 'comp-output-filename) - (comp-output-filename file) - (comp-el-to-eln-filename file))) - (defun doom--eln-file-outdated-p (file) - (when-let* ((eln-file (doom--comp-output-filename file)) - (error-file (concat eln-file ".error"))) - (push eln-file doom--expected-eln-files) - (cond ((file-exists-p eln-file) + "Check whether the corresponding .eln for `file' is outdated." + (let* ((eln-name (doom--eln-file-name file)) + (eln-file (doom--find-eln-file eln-name)) + (error-file (doom--eln-error-file eln-name))) + (cond (eln-file (when (file-newer-than-file-p file eln-file) (doom-log "%s is newer than %s" file eln-file) t)) @@ -150,18 +170,20 @@ list remains lean." (doom-log "%s is newer than %s" file error-file) t)) (t - (doom-log "%s doesn't exist" eln-file) + (doom-log "%s doesn't exist" eln-name) t)))) (defun doom--native-compile-done-h (file) - (when-let* ((file) - (eln-file (doom--comp-output-filename file)) - (error-file (concat eln-file ".error"))) - (if (file-exists-p eln-file) - (doom-log "Compiled %s" eln-file) - (make-directory (file-name-directory error-file) 'parents) - (write-region "" nil error-file) - (doom-log "Compiled %s" error-file)))) + "Callback fired when an item has finished async compilation." + (when file + (let* ((eln-name (doom--eln-file-name file)) + (eln-file (doom--eln-output-file eln-name)) + (error-file (doom--eln-error-file eln-name))) + (if (file-exists-p eln-file) + (doom-log "Compiled %s" eln-file) + (make-directory (file-name-directory error-file) 'parents) + (write-region "" nil error-file) + (doom-log "Wrote %s" error-file))))) (defun doom--native-compile-jobs () "How many async native compilation jobs are queued or in-progress." @@ -174,25 +196,39 @@ list remains lean." (defun doom--wait-for-compile-jobs () "Wait for all pending async native compilation jobs." (cl-loop for pending = (doom--native-compile-jobs) - for tick = 0 then (% (1+ tick) 15) with previous = 0 while (not (zerop pending)) - if (and (zerop tick) (/= previous pending)) do - (print! "- Waiting for %d async jobs..." pending) + if (/= previous pending) do + (print! (info "\033[KWaiting for %d async jobs...\033[1A" pending)) (setq previous pending) else do (let ((inhibit-message t)) - (sleep-for 0.1))) - ;; HACK Write .error files for any missing files which still don't exist. - ;; We'll just assume there was some kind of error... - (cl-loop for eln-file in doom--expected-eln-files - for error-file = (concat eln-file ".error") + (sleep-for 0.1)))) + +(defun doom--write-missing-eln-errors () + "Write .error files for any expected .eln files that are missing." + (cl-loop for file in doom--eln-output-expected + for eln-name = (doom--eln-file-name file) + for eln-file = (doom--eln-output-file eln-name) + for error-file = (doom--eln-error-file eln-name) unless (or (file-exists-p eln-file) - (file-exists-p error-file)) do - (make-directory (file-name-directory error-file) 'parents) - (write-region "" nil error-file) - (doom-log "Compiled %s" error-file)) - (setq doom--expected-eln-files nil)) + (file-newer-than-file-p error-file file)) + do (make-directory (file-name-directory error-file) 'parents) + (write-region "" nil error-file) + (doom-log "Wrote %s" error-file)) + (setq doom--eln-output-expected nil)) + +(defun doom--compile-site-packages () + "Queue async compilation for all non-doom Elisp files." + (when (fboundp 'native-compile-async) + (cl-loop with paths = (cl-loop for path in load-path + if (not (string-prefix-p doom-local-dir path)) + collect path) + for file in (doom-files-in paths :match "\\.el\\(\\.gz\\)?$") + if (and (file-exists-p (byte-compile-dest-file file)) + (not (doom--find-eln-file (doom--eln-file-name file)))) do + (doom-log "Compiling %s" file) + (native-compile-async file nil 'late)))) (defun doom-cli-packages-install () @@ -229,7 +265,9 @@ declaration) or dependency thereof that hasn't already been." (error (signal 'doom-package-error (list package e)))))) (progn + (doom--compile-site-packages) (doom--wait-for-compile-jobs) + (doom--write-missing-eln-errors) (print! (success "Installed %d packages") (length built))) (print! (info "No packages need to be installed")) nil)))) @@ -272,12 +310,16 @@ declaration) or dependency thereof that hasn't already been." if (or (if want-byte (doom--elc-file-outdated-p file)) (if want-native (doom--eln-file-outdated-p file))) do (setq outdated t) + (when want-native + (push file doom--eln-output-expected)) finally return outdated)) (puthash package t straight--packages-to-rebuild)))) (straight-use-package (intern package)))) (progn + (doom--compile-site-packages) (doom--wait-for-compile-jobs) - (print! (success "Rebuilt %d package(s)") (length built))) + (doom--write-missing-eln-errors) + (print! (success "\033[KRebuilt %d package(s)") (length built))) (print! (success "No packages need rebuilding")) nil)))) diff --git a/core/core.el b/core/core.el index a0b6e636d..23e9e5c13 100644 --- a/core/core.el +++ b/core/core.el @@ -285,7 +285,9 @@ config.el instead." (after! comp ;; HACK Disable native-compilation for some troublesome packages - (add-to-list 'comp-deferred-compilation-black-list "/evil-collection-vterm\\.el\\'")) + (dolist (entry (list (concat "\\`" doom-local-dir ".*/evil-collection-vterm\\.el\\'") + (concat "\\`" doom-local-dir "autoloads\\.el\\'"))) + (add-to-list 'comp-deferred-compilation-black-list entry))) ;;