From 2b1c323dbf19ce11562193dc18e41e4d93642dba Mon Sep 17 00:00:00 2001 From: Henrik Lissner Date: Fri, 16 Feb 2018 02:02:58 -0500 Subject: [PATCH] :boom: Redesign private sub-module system ~/.doom.d/modules is now a full module tree, like ~/.emacs.d/modules. Symlinks are no longer involved. Private modules can now shadow Doom modules. e.g. ~/.doom.d/modules/lang/org will take precendence over ~/.emacs.d/modules/lang/org. Also, made doom--*-load-path variables public (e.g. doom--site-load-path => doom-site-load-path), and rearranged the load-path for a 10-15% startup boost. --- core/autoload/debug.el | 3 +- core/autoload/help.el | 5 +- core/autoload/message.el | 10 ++- core/autoload/test.el | 83 +++++++----------- core/core-lib.el | 2 +- core/core-packages.el | 128 +++++++++++++++------------- core/core.el | 7 +- modules/config/private/init.el | 8 +- modules/config/private/packages.el | 15 ---- modules/lang/cc/config.el | 2 +- modules/lang/org/config.el | 2 +- modules/ui/doom-dashboard/config.el | 2 +- 12 files changed, 122 insertions(+), 145 deletions(-) delete mode 100644 modules/config/private/packages.el diff --git a/core/autoload/debug.el b/core/autoload/debug.el index 9148a225b..b72777d2b 100644 --- a/core/autoload/debug.el +++ b/core/autoload/debug.el @@ -148,7 +148,8 @@ ready to be pasted in a bug report on github." (or (ignore-errors (cl-delete-duplicates (cl-loop for file in (append (reverse (directory-files-recursively doom-core-dir "\\.elc$")) - (reverse (directory-files-recursively doom-modules-dir "\\.elc$"))) + (cl-loop for dir in doom-modules-dirs + nconc (directory-files-recursively dir "\\.elc$"))) collect (file-relative-name (file-name-directory file) doom-emacs-dir)) :test #'equal)) "n/a") diff --git a/core/autoload/help.el b/core/autoload/help.el index d7c50beb4..84b91819c 100644 --- a/core/autoload/help.el +++ b/core/autoload/help.el @@ -68,9 +68,8 @@ in." (let ((sexp (sexp-at-point))) (when (memq (car-safe sexp) '(featurep! require!)) (format "%s %s" (nth 1 sexp) (nth 2 sexp)))))) - ((and buffer-file-name - (file-in-directory-p buffer-file-name doom-modules-dir)) - (let ((module (doom-module-from-path buffer-file-name))) + (buffer-file-name + (when-let* ((module (doom-module-from-path buffer-file-name))) (format "%s %s" (car module) (cdr module))))))) (list (completing-read "Describe module: " (cl-loop for (module . sub) in (reverse (hash-table-keys doom-modules)) diff --git a/core/autoload/message.el b/core/autoload/message.el index 39af8e7b1..790981ffd 100644 --- a/core/autoload/message.el +++ b/core/autoload/message.el @@ -84,9 +84,13 @@ interactive session." (defmacro warn! (message &rest args) "Output a colored warning for the current module in the *Messages* buffer." (let ((load-file-name (or load-file-name byte-compile-current-file))) - (if (file-in-directory-p load-file-name doom-modules-dir) - `(cl-destructuring-bind (cat . mod) (doom-module-from-path ,load-file-name) - (delay-warning (format "%s %s" cat mod) (format ,message ,@args) :warning)) + (if (cl-loop for dir in doom-modules-dirs + if (file-in-directory-p load-file-name dir) + return t) + `(cl-destructuring-bind (cat . mod) + (doom-module-from-path ,load-file-name) + (delay-warning (format "%s %s" cat mod) (format ,message ,@args) + :warning)) `(delay-warning (file-relative-name load-file-name doom-emacs-dir) (format ,message ,@args) :warning)))) diff --git a/core/autoload/test.el b/core/autoload/test.el index a3f20450e..7852196e2 100644 --- a/core/autoload/test.el +++ b/core/autoload/test.el @@ -8,57 +8,33 @@ command line args following a double dash (each arg should be in the If neither is available, run all tests in all enabled modules." (interactive) + ;; ensure DOOM is initialized + (doom-initialize-packages t) (condition-case-unless-debug ex - (let (targets) - ;; ensure DOOM is initialized - (let (noninteractive) - (load (expand-file-name "core/core.el" user-emacs-directory) nil t)) - ;; collect targets - (cond ((and argv (equal (car argv) "--")) - (cl-loop for arg in (cdr argv) - if (equal arg "core") - do (push (expand-file-name "test/" doom-core-dir) targets) - else - collect - (cl-destructuring-bind (car &optional cdr) (split-string arg "/" t) - (cons (intern (concat ":" car)) - (and cdr (intern cdr)))) - into args - finally do - (setq modules args argv nil))) + (let ((target-paths + ;; Convert targets (either from MODULES or `argv') into a list of + ;; string paths, pointing to the root directory of modules + (cond ((string= (car argv) "--") ; command line + (cl-loop for arg in (cdr argv) + if (equal arg "core") collect doom-core-dir + else collect (expand-file-name arg) + finally do (setq argv nil))) - (modules - (unless (cl-loop for module in modules - unless (and (consp module) - (keywordp (car module)) - (symbolp (cdr module))) - return t) - (error "Expected a list of cons, got: %s" modules))) + (modules ; cons-cells given to MODULES + (cl-loop for (module . submodule) in modules + if (doom-module-path module submodule) + collect it)) - (t - (let (noninteractive) - (setq doom-modules (clrhash doom-modules)) - (load (expand-file-name "init.test.el" user-emacs-directory) nil t) - (setq modules (doom-module-pairs) - targets (list (expand-file-name "test/" doom-core-dir)))))) - ;; resolve targets to a list of test files and load them - (cl-loop with targets = - (append targets - (cl-loop for (module . submodule) in modules - if submodule - collect (doom-module-path module submodule "test/") - else - nconc - (cl-loop with module-name = (substring (symbol-name module) 1) - with module-path = (expand-file-name module-name doom-modules-dir) - for path in (directory-files module-path t "^\\w") - collect (expand-file-name "test/" path)))) - for dir in targets - if (file-directory-p dir) - nconc (reverse (doom-packages--files dir "\\.el$")) - into items - finally do (quiet! (mapc #'load-file items))) - ;; run all loaded tests + ((let (noninteractive) + (setq doom-modules (clrhash doom-modules)) + (load (expand-file-name "init.test.el" user-emacs-directory) nil t) + (append (list doom-core-dir) (doom-module-paths))))))) + ;; Load all the unit test files... + (dolist (path target-paths) + (when (file-directory-p (expand-file-name "test/" path)) + (dolist (test-file (reverse (doom-packages--files path "\\.el$"))) + (load test-file nil :nomessage)))) + ;; ... then run them (if noninteractive (ert-run-tests-batch-and-exit) (call-interactively #'ert-run-tests-interactively))) @@ -86,11 +62,12 @@ If neither is available, run all tests in all enabled modules." (when-let* ((after (plist-get plist :after))) (setq body `(,@body @after))) `(ert-deftest - ,(cl-loop with path = (file-relative-name (file-name-sans-extension load-file-name) - doom-emacs-dir) - for (rep . with) in '(("/test/" . "/") ("/" . ":")) - do (setq path (replace-regexp-in-string rep with path t t)) - finally return (intern (format "%s::%s" path name))) + ,(intern (format "%s::%s" + (if (file-in-directory-p load-file-name doom-core-dir) + (format "core/%s" (file-name-base load-file-name)) + (replace-regexp-in-string "^.*/modules/\\([^/]+\\)/\\([^/]+\\)/test/" "\\1/\\2:" + (file-name-sans-extension load-file-name))) + name)) () ,(if (plist-get plist :skip) `(ert-skip ,(plist-get plist :skip)) diff --git a/core/core-lib.el b/core/core-lib.el index dcf94ba33..37db56584 100644 --- a/core/core-lib.el +++ b/core/core-lib.el @@ -1,6 +1,6 @@ ;;; core-lib.el -*- lexical-binding: t; -*- -(let ((load-path doom--site-load-path)) +(let ((load-path doom-site-load-path)) (require 'subr-x) (require 'cl-lib) (require 'map)) diff --git a/core/core-packages.el b/core/core-packages.el index 55d665763..f8d327afe 100644 --- a/core/core-packages.el +++ b/core/core-packages.el @@ -58,6 +58,11 @@ this is nil after Emacs has started something is wrong.") (make-hash-table :test #'equal :size 90 :rehash-threshold 1.0) "A hash table of enabled modules. Set by `doom-initialize-modules'.") +(defvar doom-psuedo-module-dirs () + "Additional paths for modules that are outside of `doom-modules-dirs'. +`doom//reload-autoloads', `doom//byte-compile' and `doom-initialize-packages' +will include the directories in this list.") + (defvar doom-packages () "A list of enabled packages. Each element is a sublist, whose CAR is the package's name as a symbol, and whose CDR is the plist supplied to its @@ -74,24 +79,14 @@ missing) and shouldn't be deleted.") (defvar doom-reload-hook nil "A list of hooks to run when `doom/reload-load-path' is called.") -(defvar doom-extra-module-paths () - "Additional paths for modules that are outside of `doom-modules-dir'. -`doom//reload-autoloads', `doom//byte-compile' and `doom-initialize-packages' -will include the directories in this list.") +(defvar doom-site-load-path load-path + "The starting load-path, before it is altered by `doom-initialize'.") -(defvar doom--site-load-path load-path - "The load path of built in Emacs libraries.") - -(defvar doom--package-load-path () +(defvar doom-package-load-path () "The load path of package libraries installed via ELPA and QUELPA.") -(defvar doom--base-load-path - (append (list doom-core-dir doom-modules-dir) - doom--site-load-path) - "A backup of `load-path' before it was altered by `doom-initialize'. Used as a -base by `doom!' and for calculating how many packages exist.") - (defvar doom--refreshed-p nil) +(defvar doom--current-module nil) (setq package--init-file-ensured t package-user-dir (expand-file-name "elpa" doom-packages-dir) @@ -140,7 +135,7 @@ startup." ;; Called early during initialization; only use native (and cl-lib) functions! (when (or force-p (not doom-init-p)) ;; Speed things up with a `load-path' for only the bare essentials - (let ((load-path doom--site-load-path)) + (let ((load-path doom-site-load-path)) ;; Ensure core folders exist, otherwise we get errors (dolist (dir (list doom-local-dir doom-etc-dir doom-cache-dir doom-packages-dir)) (unless (file-directory-p dir) @@ -169,15 +164,18 @@ startup." (defun doom-initialize-load-path (&optional force-p) "Populates `load-path', if it hasn't already been. If FORCE-P is non-nil, do it anyway." - (when (or force-p (not doom--package-load-path)) + (when (or force-p (not doom-package-load-path)) ;; We could let `package-initialize' fill `load-path', but it does more than ;; that alone (like load autoload files). If you want something prematurely ;; optimizated right, ya gotta do it yourself. ;; ;; Also, in some edge cases involving package initialization during a ;; non-interactive session, `package-initialize' fails to fill `load-path'. - (setq doom--package-load-path (directory-files package-user-dir t "^[^.]" t) - load-path (append doom--package-load-path doom--base-load-path)))) + (setq doom-package-load-path (directory-files package-user-dir t "^[^.]" t) + load-path (append doom-package-load-path + doom-site-load-path + doom-core-dir + doom-modules-dirs)))) (defun doom-initialize-autoloads () "Ensures that `doom-autoload-file' exists and is loaded. Otherwise run @@ -218,7 +216,13 @@ This aggressively reloads core autoload files." (when (or force-p (not doom-packages)) (setq doom-packages nil) (_load (expand-file-name "packages.el" doom-core-dir)) - (mapc #'_load (doom-module-paths "packages.el")))))) + (cl-loop for key being the hash-keys of doom-modules + if (doom-module-path (car key) (cdr key) "packages.el") + do (doom-module-load-file (car key) (cdr key) it)) + (cl-loop for dir in doom-psuedo-module-dirs + for path = (expand-file-name "packages.el" dir) + if (file-exists-p path) + do (_load path)))))) (defun doom-module-path (module submodule &optional file) "Get the full path to a module: e.g. :lang emacs-lisp maps to @@ -227,31 +231,29 @@ This aggressively reloads core autoload files." (setq module (substring (symbol-name module) 1))) (when (symbolp submodule) (setq submodule (symbol-name submodule))) - (expand-file-name (concat module "/" submodule "/" file) - doom-modules-dir)) + (cl-loop for default-directory in doom-modules-dirs + for path = (concat module "/" submodule "/" file) + if (file-exists-p path) + return (expand-file-name path))) -(defun doom-module-from-path (path) +(defun doom-module-from-path (&optional path) "Get module cons cell (MODULE . SUBMODULE) for PATH, if possible." - (save-match-data - (setq path (file-truename path)) - (when (string-match "/modules/\\([^/]+\\)/\\([^/]+\\)/.*$" path) - (when-let* ((module (match-string 1 path)) - (submodule (match-string 2 path))) - (cons (intern (concat ":" module)) - (intern submodule)))))) + (or doom--current-module + (save-match-data + (setq path (file-truename path)) + (when (string-match "/modules/\\([^/]+\\)/\\([^/]+\\)/.*$" path) + (when-let* ((module (match-string 1 path)) + (submodule (match-string 2 path))) + (cons (intern (concat ":" module)) + (intern submodule))))))) (defun doom-module-paths (&optional append-file) "Returns a list of absolute file paths to activated modules, with APPEND-FILE added, if the file exists." - (let ((fn (if append-file #'file-exists-p #'file-directory-p))) - (append (cl-loop for (module . submodule) in (doom-module-pairs) - for path = (doom-module-path module submodule append-file) - if (funcall fn path) - collect path) - (cl-loop for dir in doom-extra-module-paths - for path = (if append-file (expand-file-name append-file dir) dir) - if (funcall fn path) - collect path)))) + (append (cl-loop for key being the hash-keys of doom-modules + if (doom-module-path (car key) (cdr key) append-file) + collect it) + (cl-remove-if-not #'file-directory-p doom-psuedo-module-dirs))) (defun doom-module-flags (module submodule) "Returns a list of flags provided for MODULE SUBMODULE." @@ -281,17 +283,22 @@ Used by `require!' and `depends-on!'." (defun doom-module-pairs () "Returns `doom-modules' as a list of (MODULE . SUBMODULE) cons cells." - (unless (hash-table-p doom-modules) - (error "doom-modules is uninitialized")) (cl-loop for key being the hash-keys of doom-modules collect key)) +(defun doom-module-load-file (module submodule file &optional path) + "Load FILE in MODULE/SUBMODULE. If PATH is specified, look for FILE in PATH." + (unless (or path (file-name-absolute-p file)) + (setq path (doom-module-path module submodule file))) + (let ((doom--current-module (cons module submodule))) + (load (expand-file-name file path) :noerror (not doom-debug-mode)))) + (defun doom-packages--display-benchmark () (message "Doom loaded %s packages across %d modules in %.03fs" ;; Certainly imprecise, especially where custom additions to ;; load-path are concerned, but I don't mind a [small] margin of ;; error in the plugin count in exchange for faster startup. - (length doom--package-load-path) + (length doom-package-load-path) (hash-table-size doom-modules) (setq doom-init-time (float-time (time-subtract (current-time) before-init-time))))) @@ -306,17 +313,16 @@ Used by `require!' and `depends-on!'." "Bootstraps DOOM Emacs and its modules. MODULES is an malformed plist of modules to load." - (let (init-forms config-forms module) + (let (init-forms config-forms module file-name-handler-alist) (dolist (m modules) (cond ((keywordp m) (setq module m)) ((not module) (error "No namespace specified in `doom!' for %s" m)) - (t - (let ((submodule (if (listp m) (car m) m)) + ((let ((submodule (if (listp m) (car m) m)) (flags (if (listp m) (cdr m)))) (doom-module-enable module submodule flags) (let ((path (doom-module-path module submodule))) - (push `(load! init ,path t) init-forms) - (push `(load! config ,path t) config-forms)))))) + (push `(doom-module-load-file ,module ',submodule "init" ,path) init-forms) + (push `(doom-module-load-file ,module ',submodule "config" ,path) config-forms)))))) `(let (file-name-handler-alist) (setq doom-modules ',doom-modules) ,@(nreverse init-forms) @@ -408,10 +414,8 @@ The module is only loaded once. If RELOAD-P is non-nil, load it again." (if (file-directory-p module-path) `(condition-case-unless-debug ex (progn - (load! init ,module-path t) - (if after-init-time - (load! config ,module-path t) - (add-hook! 'doom-init-hook (load! config ,module-path t)))) + (doom-module-load-file ,module ',submodule "init" ,module-path) + (doom-module-load-file ,module ',submodule "config" ,module-path)) ('error (lwarn 'doom-modules :error "%s in '%s %s' -> %s" @@ -544,7 +548,7 @@ call `doom//reload-load-path' remotely (through emacsclient)." (server-eval-at server-name '(doom//reload-load-path)))) ((let ((noninteractive t)) (doom-initialize-load-path t) - (message "%d packages reloaded" (length doom--package-load-path)) + (message "%d packages reloaded" (length doom-package-load-path)) (run-hooks 'doom-reload-hook))))) (defun doom//reload-autoloads () @@ -570,7 +574,7 @@ This should be run whenever init.el or an autoload file is modified. Running (let ((targets (file-expand-wildcards (expand-file-name "autoload/*.el" doom-core-dir)))) - (dolist (path (doom-module-paths)) + (dolist (path (append doom-psuedo-module-dirs (doom-module-paths))) (let ((auto-dir (expand-file-name "autoload" path)) (auto-file (expand-file-name "autoload.el" path))) (when (file-exists-p auto-file) @@ -652,8 +656,15 @@ If RECOMPILE-P is non-nil, only recompile out-of-date files." nconc (nreverse (doom-packages--files doom-core-dir "\\.el$")) else if (file-directory-p target) nconc (nreverse (doom-packages--files target "\\.el$")) - else if (file-directory-p (expand-file-name target doom-modules-dir)) - nconc (nreverse (doom-packages--files (expand-file-name target doom-modules-dir) "\\.el$")) + else if (cl-loop for dir in doom-psuedo-module-dirs + if (file-in-directory-p target dir) + return dir) + nconc (nreverse (doom-packages--files it "\\.el$")) + else if (string-match "^\\([^/]+\\)/\\([^/]+\\)$" target) + nconc (nreverse (doom-packages--files + (doom-module-path (intern (format ":%s" (match-string 1 target))) + (intern (match-string 2 target))) + "\\.el$")) else if (file-exists-p target) collect target finally do (setq argv nil))) @@ -663,7 +674,7 @@ If RECOMPILE-P is non-nil, only recompile out-of-date files." (push (expand-file-name "init.el" doom-emacs-dir) compile-targets) (condition-case ex (progn - (dolist (target compile-targets) + (dolist (target (cl-delete-duplicates compile-targets :test #'string= :key #'file-truename)) (when (or (not recompile-p) (let ((elc-file (byte-compile-dest-file target))) (and (file-exists-p elc-file) @@ -671,7 +682,9 @@ If RECOMPILE-P is non-nil, only recompile out-of-date files." (let ((result (if (doom-packages--read-if-cookies target) (byte-compile-file target) 'no-byte-compile)) - (short-name (file-relative-name target doom-emacs-dir))) + (short-name (if (file-in-directory-p target doom-emacs-dir) + (file-relative-name target doom-emacs-dir) + (abbreviate-file-name target)))) (cl-incf (cond ((eq result 'no-byte-compile) (message! (dark (white "⚠ Ignored %s" short-name))) @@ -727,8 +740,7 @@ compiled packages.'" (let ((targets (append (list (expand-file-name "init.elc" doom-emacs-dir)) (doom-packages--files doom-core-dir "\\.elc$") - (doom-packages--files doom-modules-dir "\\.elc$") - (cl-loop for dir in doom-extra-module-paths + (cl-loop for dir in (append doom-modules-dirs doom-psuedo-module-dirs) if (file-directory-p dir) nconc (doom-packages--files dir "\\.elc$")))) (default-directory doom-emacs-dir)) diff --git a/core/core.el b/core/core.el index e5b5ce7ad..610cb3336 100644 --- a/core/core.el +++ b/core/core.el @@ -32,7 +32,10 @@ line or use --debug-init to enable this.") "Where essential files are stored.") (defvar doom-modules-dir (concat doom-emacs-dir "modules/") - "Where configuration modules are stored.") + "The main directory where Doom modules are stored.") + +(defvar doom-modules-dirs (list doom-modules-dir) + "A list of module root directories. Order determines priority.") (defvar doom-local-dir (concat doom-emacs-dir ".local/") "Root directory for local Emacs files. Use this as permanent storage for files @@ -159,7 +162,7 @@ ability to invoke the debugger in debug mode." (load (concat doom-core-dir "core-packages") nil t) (setq load-path (eval-when-compile (doom-initialize t) (doom-initialize-load-path t)) - doom--package-load-path (eval-when-compile doom--package-load-path)) + doom-package-load-path (eval-when-compile doom-package-load-path)) (load! core-lib) (load! core-os) ; consistent behavior across OSes diff --git a/modules/config/private/init.el b/modules/config/private/init.el index e361955fe..762064d97 100644 --- a/modules/config/private/init.el +++ b/modules/config/private/init.el @@ -7,15 +7,11 @@ "The directory that serves as the root of your external private config for Doom Emacs.") -(defvar +private-symlink-path - (expand-file-name "private" doom-modules-dir) - "Where the place the symbolic link to the private modules directory.") - ;; Ensure `doom//reload-autoloads', `doom//byte-compile' and ;; `doom-initialize-packages' all include this module in their operations. -(add-to-list 'doom-extra-module-paths +private-config-path) +(add-to-list 'doom-psuedo-module-dirs +private-config-path) +(add-to-list 'doom-modules-dirs (expand-file-name "modules/" +private-config-path)) ;; (load (expand-file-name "init.el" +private-config-path) 'noerror 'nomessage) - diff --git a/modules/config/private/packages.el b/modules/config/private/packages.el deleted file mode 100644 index 0544484ae..000000000 --- a/modules/config/private/packages.el +++ /dev/null @@ -1,15 +0,0 @@ -;; -*- no-byte-compile: t; -*- -;;; config/private/packages.el - -(let ((modules-path (expand-file-name "modules/" +private-config-path))) - (when (file-directory-p modules-path) - ;; automatically symlinks your private modules to `doom-modules-dir'. - (if (and (file-directory-p +private-symlink-path) - (not (file-symlink-p +private-symlink-path))) - (lwarn "config/private" :warning - "modules/%s already exists; can't create symlink" +private-symlink-path) - (make-symbolic-link modules-path +private-symlink-path t)))) - -;; -(load (expand-file-name "packages.el" +private-config-path) - 'noerror 'nomessage) diff --git a/modules/lang/cc/config.el b/modules/lang/cc/config.el index c89d83e72..502bc82dc 100644 --- a/modules/lang/cc/config.el +++ b/modules/lang/cc/config.el @@ -233,7 +233,7 @@ compilation database is present in the project.") ;; later, so we un-byte-compile it before we load it. (eval-when-compile (when (>= emacs-major-version 26) - (when-let* ((elc-file (locate-library "rtags.elc" t doom--package-load-path))) + (when-let* ((elc-file (locate-library "rtags.elc" t doom-package-load-path))) (delete-file elc-file)))) :config (setq rtags-display-result-backend 'ivy)) diff --git a/modules/lang/org/config.el b/modules/lang/org/config.el index eb17a0b11..c37f3c5e5 100644 --- a/modules/lang/org/config.el +++ b/modules/lang/org/config.el @@ -198,8 +198,8 @@ unfold to point on startup." (def-org-file-link! "org" +org-dir) (def-org-file-link! "doom" doom-emacs-dir) - (def-org-file-link! "doom-module" doom-modules-dir) (def-org-file-link! "doom-docs" doom-docs-dir) + (def-org-file-link! "doom-modules" doom-modules-dir) ;; Update UI when theme is changed (add-hook 'doom-init-theme-hook #'+org|setup-ui)) diff --git a/modules/ui/doom-dashboard/config.el b/modules/ui/doom-dashboard/config.el index 190da6f5f..374a0c6f8 100644 --- a/modules/ui/doom-dashboard/config.el +++ b/modules/ui/doom-dashboard/config.el @@ -263,7 +263,7 @@ controlled by `+doom-dashboard-pwd-policy'." (+doom-dashboard--center +doom-dashboard--width (format "Loaded %d packages in %d modules in %.02fs" - (length doom--package-load-path) + (length doom-package-load-path) (hash-table-size doom-modules) doom-init-time)) 'face 'font-lock-comment-face)