💥 Redesign private sub-module system

~/.doom.d/modules is now a full module tree, like ~/.emacs.d/modules.
Symlinks are no longer involved.

Private modules can now shadow Doom modules. e.g.
~/.doom.d/modules/lang/org will take precendence over
~/.emacs.d/modules/lang/org.

Also, made doom--*-load-path variables public (e.g. doom--site-load-path
=> doom-site-load-path), and rearranged the load-path for a 10-15%
startup boost.

core/autoload/debug.el

@@ -148,7 +148,8 @@ ready to be pasted in a bug report on github."
      (or (ignore-errors
            (cl-delete-duplicates
             (cl-loop for file in (append (reverse (directory-files-recursively doom-core-dir "\\.elc$"))
-                                         (reverse (directory-files-recursively doom-modules-dir "\\.elc$")))
+                                         (cl-loop for dir in doom-modules-dirs
+                                                  nconc (directory-files-recursively dir "\\.elc$")))
                      collect (file-relative-name (file-name-directory file) doom-emacs-dir))
             :test #'equal))
          "n/a")

core/autoload/help.el

@@ -68,9 +68,8 @@ in."
                    (let ((sexp (sexp-at-point)))
                      (when (memq (car-safe sexp) '(featurep! require!))
                        (format "%s %s" (nth 1 sexp) (nth 2 sexp))))))
-                ((and buffer-file-name
+                (buffer-file-name
-                      (file-in-directory-p buffer-file-name doom-modules-dir))
+                 (when-let* ((module (doom-module-from-path buffer-file-name)))
-                 (let ((module (doom-module-from-path buffer-file-name)))
                    (format "%s %s" (car module) (cdr module)))))))
      (list (completing-read "Describe module: "
                             (cl-loop for (module . sub) in (reverse (hash-table-keys doom-modules))

core/autoload/message.el

@@ -84,9 +84,13 @@ interactive session."
 (defmacro warn! (message &rest args)
   "Output a colored warning for the current module in the *Messages* buffer."
   (let ((load-file-name (or load-file-name byte-compile-current-file)))
-    (if (file-in-directory-p load-file-name doom-modules-dir)
+    (if (cl-loop for dir in doom-modules-dirs
-        `(cl-destructuring-bind (cat . mod) (doom-module-from-path ,load-file-name)
+                 if (file-in-directory-p load-file-name dir)
-           (delay-warning (format "%s %s" cat mod) (format ,message ,@args) :warning))
+                 return t)
+        `(cl-destructuring-bind (cat . mod)
+             (doom-module-from-path ,load-file-name)
+           (delay-warning (format "%s %s" cat mod) (format ,message ,@args)
+                          :warning))
       `(delay-warning (file-relative-name load-file-name doom-emacs-dir)
                       (format ,message ,@args) :warning))))
 

core/autoload/test.el

@@ -8,57 +8,33 @@ command line args following a double dash (each arg should be in the
 
 If neither is available, run all tests in all enabled modules."
   (interactive)
-  (condition-case-unless-debug ex
-      (let (targets)
   ;; ensure DOOM is initialized
-        (let (noninteractive)
+  (doom-initialize-packages t)
-          (load (expand-file-name "core/core.el" user-emacs-directory) nil t))
+  (condition-case-unless-debug ex
-        ;; collect targets
+      (let ((target-paths
-        (cond ((and argv (equal (car argv) "--"))
+             ;; Convert targets (either from MODULES or `argv') into a list of
+             ;; string paths, pointing to the root directory of modules
+             (cond ((string= (car argv) "--") ; command line
                     (cl-loop for arg in (cdr argv)
-                        if (equal arg "core")
+                             if (equal arg "core") collect doom-core-dir
-                         do (push (expand-file-name "test/" doom-core-dir) targets)
+                             else collect (expand-file-name arg)
-                        else
+                             finally do (setq argv nil)))
-                         collect
-                         (cl-destructuring-bind (car &optional cdr) (split-string arg "/" t)
-                           (cons (intern (concat ":" car))
-                                 (and cdr (intern cdr))))
-                         into args
-                        finally do
-                        (setq modules args argv nil)))
 
-              (modules
+                   (modules ; cons-cells given to MODULES
-               (unless (cl-loop for module in modules
+                    (cl-loop for (module . submodule) in modules
-                                unless (and (consp module)
+                             if (doom-module-path module submodule)
-                                            (keywordp (car module))
+                             collect it))
-                                            (symbolp (cdr module)))
-                                return t)
-                 (error "Expected a list of cons, got: %s" modules)))
 
-              (t
+                   ((let (noninteractive)
-               (let (noninteractive)
                       (setq doom-modules (clrhash doom-modules))
                       (load (expand-file-name "init.test.el" user-emacs-directory) nil t)
-                 (setq modules (doom-module-pairs)
+                      (append (list doom-core-dir) (doom-module-paths)))))))
-                       targets (list (expand-file-name "test/" doom-core-dir))))))
+        ;; Load all the unit test files...
-        ;; resolve targets to a list of test files and load them
+        (dolist (path target-paths)
-        (cl-loop with targets =
+          (when (file-directory-p (expand-file-name "test/" path))
-                 (append targets
+            (dolist (test-file (reverse (doom-packages--files path "\\.el$")))
-                         (cl-loop for (module . submodule) in modules
+              (load test-file nil :nomessage))))
-                                  if submodule
+        ;; ... then run them
-                                  collect (doom-module-path module submodule "test/")
-                                  else
-                                  nconc
-                                  (cl-loop with module-name = (substring (symbol-name module) 1)
-                                           with module-path = (expand-file-name module-name doom-modules-dir)
-                                           for path in (directory-files module-path t "^\\w")
-                                           collect (expand-file-name "test/" path))))
-                 for dir in targets
-                 if (file-directory-p dir)
-                  nconc (reverse (doom-packages--files dir "\\.el$"))
-                  into items
-                 finally do (quiet! (mapc #'load-file items)))
-        ;; run all loaded tests
         (if noninteractive
             (ert-run-tests-batch-and-exit)
           (call-interactively #'ert-run-tests-interactively)))
@@ -86,11 +62,12 @@ If neither is available, run all tests in all enabled modules."
     (when-let* ((after (plist-get plist :after)))
       (setq body `(,@body @after)))
     `(ert-deftest
-         ,(cl-loop with path = (file-relative-name (file-name-sans-extension load-file-name)
+         ,(intern (format "%s::%s"
-                                                   doom-emacs-dir)
+                          (if (file-in-directory-p load-file-name doom-core-dir)
-                   for (rep . with) in '(("/test/" . "/") ("/" . ":"))
+                              (format "core/%s" (file-name-base load-file-name))
-                   do (setq path (replace-regexp-in-string rep with path t t))
+                            (replace-regexp-in-string "^.*/modules/\\([^/]+\\)/\\([^/]+\\)/test/" "\\1/\\2:"
-                   finally return (intern (format "%s::%s" path name)))
+                                                      (file-name-sans-extension load-file-name)))
+                          name))
          ()
        ,(if (plist-get plist :skip)
             `(ert-skip ,(plist-get plist :skip))

core/core-lib.el

@@ -1,6 +1,6 @@
 ;;; core-lib.el -*- lexical-binding: t; -*-
 
-(let ((load-path doom--site-load-path))
+(let ((load-path doom-site-load-path))
   (require 'subr-x)
   (require 'cl-lib)
   (require 'map))

core/core-packages.el

@@ -58,6 +58,11 @@ this is nil after Emacs has started something is wrong.")
   (make-hash-table :test #'equal :size 90 :rehash-threshold 1.0)
   "A hash table of enabled modules. Set by `doom-initialize-modules'.")
 
+(defvar doom-psuedo-module-dirs ()
+  "Additional paths for modules that are outside of `doom-modules-dirs'.
+`doom//reload-autoloads', `doom//byte-compile' and `doom-initialize-packages'
+will include the directories in this list.")
+
 (defvar doom-packages ()
   "A list of enabled packages. Each element is a sublist, whose CAR is the
 package's name as a symbol, and whose CDR is the plist supplied to its
@@ -74,24 +79,14 @@ missing) and shouldn't be deleted.")
 (defvar doom-reload-hook nil
   "A list of hooks to run when `doom/reload-load-path' is called.")
 
-(defvar doom-extra-module-paths ()
+(defvar doom-site-load-path load-path
-  "Additional paths for modules that are outside of `doom-modules-dir'.
+  "The starting load-path, before it is altered by `doom-initialize'.")
-`doom//reload-autoloads', `doom//byte-compile' and `doom-initialize-packages'
-will include the directories in this list.")
 
-(defvar doom--site-load-path load-path
+(defvar doom-package-load-path ()
-  "The load path of built in Emacs libraries.")
-
-(defvar doom--package-load-path ()
   "The load path of package libraries installed via ELPA and QUELPA.")
 
-(defvar doom--base-load-path
-  (append (list doom-core-dir doom-modules-dir)
-          doom--site-load-path)
-  "A backup of `load-path' before it was altered by `doom-initialize'. Used as a
-base by `doom!' and for calculating how many packages exist.")
-
 (defvar doom--refreshed-p nil)
+(defvar doom--current-module nil)
 
 (setq package--init-file-ensured t
       package-user-dir (expand-file-name "elpa" doom-packages-dir)
@@ -140,7 +135,7 @@ startup."
   ;; Called early during initialization; only use native (and cl-lib) functions!
   (when (or force-p (not doom-init-p))
     ;; Speed things up with a `load-path' for only the bare essentials
-    (let ((load-path doom--site-load-path))
+    (let ((load-path doom-site-load-path))
       ;; Ensure core folders exist, otherwise we get errors
       (dolist (dir (list doom-local-dir doom-etc-dir doom-cache-dir doom-packages-dir))
         (unless (file-directory-p dir)
@@ -169,15 +164,18 @@ startup."
 (defun doom-initialize-load-path (&optional force-p)
   "Populates `load-path', if it hasn't already been. If FORCE-P is non-nil, do
 it anyway."
-  (when (or force-p (not doom--package-load-path))
+  (when (or force-p (not doom-package-load-path))
     ;; We could let `package-initialize' fill `load-path', but it does more than
     ;; that alone (like load autoload files). If you want something prematurely
     ;; optimizated right, ya gotta do it yourself.
     ;;
     ;; Also, in some edge cases involving package initialization during a
     ;; non-interactive session, `package-initialize' fails to fill `load-path'.
-    (setq doom--package-load-path (directory-files package-user-dir t "^[^.]" t)
+    (setq doom-package-load-path (directory-files package-user-dir t "^[^.]" t)
-          load-path (append doom--package-load-path doom--base-load-path))))
+          load-path (append doom-package-load-path
+                            doom-site-load-path
+                            doom-core-dir
+                            doom-modules-dirs))))
 
 (defun doom-initialize-autoloads ()
   "Ensures that `doom-autoload-file' exists and is loaded. Otherwise run
@@ -218,7 +216,13 @@ This aggressively reloads core autoload files."
       (when (or force-p (not doom-packages))
         (setq doom-packages nil)
         (_load (expand-file-name "packages.el" doom-core-dir))
-        (mapc #'_load (doom-module-paths "packages.el"))))))
+        (cl-loop for key being the hash-keys of doom-modules
+                 if (doom-module-path (car key) (cdr key) "packages.el")
+                 do (doom-module-load-file (car key) (cdr key) it))
+        (cl-loop for dir in doom-psuedo-module-dirs
+                 for path = (expand-file-name "packages.el" dir)
+                 if (file-exists-p path)
+                 do (_load path))))))
 
 (defun doom-module-path (module submodule &optional file)
   "Get the full path to a module: e.g. :lang emacs-lisp maps to
@@ -227,31 +231,29 @@ This aggressively reloads core autoload files."
     (setq module (substring (symbol-name module) 1)))
   (when (symbolp submodule)
     (setq submodule (symbol-name submodule)))
-  (expand-file-name (concat module "/" submodule "/" file)
+  (cl-loop for default-directory in doom-modules-dirs
-                    doom-modules-dir))
+           for path = (concat module "/" submodule "/" file)
+           if (file-exists-p path)
+           return (expand-file-name path)))
 
-(defun doom-module-from-path (path)
+(defun doom-module-from-path (&optional path)
   "Get module cons cell (MODULE . SUBMODULE) for PATH, if possible."
+    (or doom--current-module
         (save-match-data
           (setq path (file-truename path))
           (when (string-match "/modules/\\([^/]+\\)/\\([^/]+\\)/.*$" path)
             (when-let* ((module (match-string 1 path))
                         (submodule (match-string 2 path)))
               (cons (intern (concat ":" module))
-              (intern submodule))))))
+                    (intern submodule)))))))
 
 (defun doom-module-paths (&optional append-file)
   "Returns a list of absolute file paths to activated modules, with APPEND-FILE
 added, if the file exists."
-  (let ((fn (if append-file #'file-exists-p #'file-directory-p)))
+  (append (cl-loop for key being the hash-keys of doom-modules
-    (append (cl-loop for (module . submodule) in (doom-module-pairs)
+                   if (doom-module-path (car key) (cdr key) append-file)
-                     for path = (doom-module-path module submodule append-file)
+                   collect it)
-                     if (funcall fn path)
+          (cl-remove-if-not #'file-directory-p doom-psuedo-module-dirs)))
-                     collect path)
-            (cl-loop for dir in doom-extra-module-paths
-                     for path = (if append-file (expand-file-name append-file dir) dir)
-                     if (funcall fn path)
-                     collect path))))
 
 (defun doom-module-flags (module submodule)
   "Returns a list of flags provided for MODULE SUBMODULE."
@@ -281,17 +283,22 @@ Used by `require!' and `depends-on!'."
 
 (defun doom-module-pairs ()
   "Returns `doom-modules' as a list of (MODULE . SUBMODULE) cons cells."
-  (unless (hash-table-p doom-modules)
-    (error "doom-modules is uninitialized"))
   (cl-loop for key being the hash-keys of doom-modules
            collect key))
 
+(defun doom-module-load-file (module submodule file &optional path)
+  "Load FILE in MODULE/SUBMODULE. If PATH is specified, look for FILE in PATH."
+  (unless (or path (file-name-absolute-p file))
+    (setq path (doom-module-path module submodule file)))
+  (let ((doom--current-module (cons module submodule)))
+    (load (expand-file-name file path) :noerror (not doom-debug-mode))))
+
 (defun doom-packages--display-benchmark ()
   (message "Doom loaded %s packages across %d modules in %.03fs"
            ;; Certainly imprecise, especially where custom additions to
            ;; load-path are concerned, but I don't mind a [small] margin of
            ;; error in the plugin count in exchange for faster startup.
-           (length doom--package-load-path)
+           (length doom-package-load-path)
            (hash-table-size doom-modules)
            (setq doom-init-time (float-time (time-subtract (current-time) before-init-time)))))
 
@@ -306,17 +313,16 @@ Used by `require!' and `depends-on!'."
   "Bootstraps DOOM Emacs and its modules.
 
 MODULES is an malformed plist of modules to load."
-  (let (init-forms config-forms module)
+  (let (init-forms config-forms module file-name-handler-alist)
     (dolist (m modules)
       (cond ((keywordp m) (setq module m))
             ((not module) (error "No namespace specified in `doom!' for %s" m))
-            (t
+            ((let ((submodule (if (listp m) (car m) m))
-             (let ((submodule (if (listp m) (car m) m))
                    (flags     (if (listp m) (cdr m))))
                (doom-module-enable module submodule flags)
                (let ((path (doom-module-path module submodule)))
-                 (push `(load! init ,path t) init-forms)
+                 (push `(doom-module-load-file ,module ',submodule "init" ,path) init-forms)
-                 (push `(load! config ,path t) config-forms))))))
+                 (push `(doom-module-load-file ,module ',submodule "config" ,path) config-forms))))))
     `(let (file-name-handler-alist)
        (setq doom-modules ',doom-modules)
        ,@(nreverse init-forms)
@@ -408,10 +414,8 @@ The module is only loaded once. If RELOAD-P is non-nil, load it again."
       (if (file-directory-p module-path)
           `(condition-case-unless-debug ex
                (progn
-                 (load! init ,module-path t)
+                 (doom-module-load-file ,module ',submodule "init" ,module-path)
-                 (if after-init-time
+                 (doom-module-load-file ,module ',submodule "config" ,module-path))
-                     (load! config ,module-path t)
-                   (add-hook! 'doom-init-hook (load! config ,module-path t))))
              ('error
               (lwarn 'doom-modules :error
                      "%s in '%s %s' -> %s"
@@ -544,7 +548,7 @@ call `doom//reload-load-path' remotely (through emacsclient)."
            (server-eval-at server-name '(doom//reload-load-path))))
         ((let ((noninteractive t))
            (doom-initialize-load-path t)
-           (message "%d packages reloaded" (length doom--package-load-path))
+           (message "%d packages reloaded" (length doom-package-load-path))
            (run-hooks 'doom-reload-hook)))))
 
 (defun doom//reload-autoloads ()
@@ -570,7 +574,7 @@ This should be run whenever init.el or an autoload file is modified. Running
     (let ((targets
            (file-expand-wildcards
             (expand-file-name "autoload/*.el" doom-core-dir))))
-      (dolist (path (doom-module-paths))
+      (dolist (path (append doom-psuedo-module-dirs (doom-module-paths)))
         (let ((auto-dir  (expand-file-name "autoload" path))
               (auto-file (expand-file-name "autoload.el" path)))
           (when (file-exists-p auto-file)
@@ -652,8 +656,15 @@ If RECOMPILE-P is non-nil, only recompile out-of-date files."
                         nconc (nreverse (doom-packages--files doom-core-dir "\\.el$"))
                        else if (file-directory-p target)
                         nconc (nreverse (doom-packages--files target "\\.el$"))
-                       else if (file-directory-p (expand-file-name target doom-modules-dir))
+                       else if (cl-loop for dir in doom-psuedo-module-dirs
-                        nconc (nreverse (doom-packages--files (expand-file-name target doom-modules-dir) "\\.el$"))
+                                        if (file-in-directory-p target dir)
+                                        return dir)
+                        nconc (nreverse (doom-packages--files it "\\.el$"))
+                       else if (string-match "^\\([^/]+\\)/\\([^/]+\\)$" target)
+                        nconc (nreverse (doom-packages--files
+                                         (doom-module-path (intern (format ":%s" (match-string 1 target)))
+                                                           (intern (match-string 2 target)))
+                                         "\\.el$"))
                        else if (file-exists-p target)
                         collect target
                        finally do (setq argv nil)))
@@ -663,7 +674,7 @@ If RECOMPILE-P is non-nil, only recompile out-of-date files."
           (push (expand-file-name "init.el" doom-emacs-dir) compile-targets)
           (condition-case ex
               (progn
-                (dolist (target compile-targets)
+                (dolist (target (cl-delete-duplicates compile-targets :test #'string= :key #'file-truename))
                   (when (or (not recompile-p)
                             (let ((elc-file (byte-compile-dest-file target)))
                               (and (file-exists-p elc-file)
@@ -671,7 +682,9 @@ If RECOMPILE-P is non-nil, only recompile out-of-date files."
                     (let ((result (if (doom-packages--read-if-cookies target)
                                       (byte-compile-file target)
                                     'no-byte-compile))
-                          (short-name (file-relative-name target doom-emacs-dir)))
+                          (short-name (if (file-in-directory-p target doom-emacs-dir)
+                                          (file-relative-name target doom-emacs-dir)
+                                        (abbreviate-file-name target))))
                       (cl-incf
                        (cond ((eq result 'no-byte-compile)
                               (message! (dark (white "⚠ Ignored %s" short-name)))
@@ -727,8 +740,7 @@ compiled packages.'"
   (let ((targets
          (append (list (expand-file-name "init.elc" doom-emacs-dir))
                  (doom-packages--files doom-core-dir "\\.elc$")
-                 (doom-packages--files doom-modules-dir "\\.elc$")
+                 (cl-loop for dir in (append doom-modules-dirs doom-psuedo-module-dirs)
-                 (cl-loop for dir in doom-extra-module-paths
                           if (file-directory-p dir)
                           nconc (doom-packages--files dir "\\.elc$"))))
         (default-directory doom-emacs-dir))

core/core.el

@@ -32,7 +32,10 @@ line or use --debug-init to enable this.")
   "Where essential files are stored.")
 
 (defvar doom-modules-dir (concat doom-emacs-dir "modules/")
-  "Where configuration modules are stored.")
+  "The main directory where Doom modules are stored.")
+
+(defvar doom-modules-dirs (list doom-modules-dir)
+  "A list of module root directories. Order determines priority.")
 
 (defvar doom-local-dir (concat doom-emacs-dir ".local/")
   "Root directory for local Emacs files. Use this as permanent storage for files
@@ -159,7 +162,7 @@ ability to invoke the debugger in debug mode."
   (load (concat doom-core-dir "core-packages") nil t)
   (setq load-path (eval-when-compile (doom-initialize t)
                                      (doom-initialize-load-path t))
-        doom--package-load-path (eval-when-compile doom--package-load-path))
+        doom-package-load-path (eval-when-compile doom-package-load-path))
 
   (load! core-lib)
   (load! core-os) ; consistent behavior across OSes

modules/config/private/init.el

@@ -7,15 +7,11 @@
   "The directory that serves as the root of your external private config for
 Doom Emacs.")
 
-(defvar +private-symlink-path
-  (expand-file-name "private" doom-modules-dir)
-  "Where the place the symbolic link to the private modules directory.")
-
 ;; Ensure `doom//reload-autoloads', `doom//byte-compile' and
 ;; `doom-initialize-packages' all include this module in their operations.
-(add-to-list 'doom-extra-module-paths +private-config-path)
+(add-to-list 'doom-psuedo-module-dirs +private-config-path)
+(add-to-list 'doom-modules-dirs (expand-file-name "modules/" +private-config-path))
 
 ;;
 (load (expand-file-name "init.el" +private-config-path)
       'noerror 'nomessage)
-

modules/config/private/packages.el

@@ -1,15 +0,0 @@
-;; -*- no-byte-compile: t; -*-
-;;; config/private/packages.el
-
-(let ((modules-path (expand-file-name "modules/" +private-config-path)))
-  (when (file-directory-p modules-path)
-    ;; automatically symlinks your private modules to `doom-modules-dir'.
-    (if (and (file-directory-p +private-symlink-path)
-             (not (file-symlink-p +private-symlink-path)))
-        (lwarn "config/private" :warning
-               "modules/%s already exists; can't create symlink" +private-symlink-path)
-      (make-symbolic-link modules-path +private-symlink-path t))))
-
-;;
-(load (expand-file-name "packages.el" +private-config-path)
-      'noerror 'nomessage)

modules/lang/cc/config.el

@@ -233,7 +233,7 @@ compilation database is present in the project.")
   ;; later, so we un-byte-compile it before we load it.
   (eval-when-compile
     (when (>= emacs-major-version 26)
-      (when-let* ((elc-file (locate-library "rtags.elc" t doom--package-load-path)))
+      (when-let* ((elc-file (locate-library "rtags.elc" t doom-package-load-path)))
         (delete-file elc-file))))
   :config (setq rtags-display-result-backend 'ivy))
 

modules/lang/org/config.el

@@ -198,8 +198,8 @@ unfold to point on startup."
 
   (def-org-file-link! "org" +org-dir)
   (def-org-file-link! "doom" doom-emacs-dir)
-  (def-org-file-link! "doom-module" doom-modules-dir)
   (def-org-file-link! "doom-docs" doom-docs-dir)
+  (def-org-file-link! "doom-modules" doom-modules-dir)
 
   ;; Update UI when theme is changed
   (add-hook 'doom-init-theme-hook #'+org|setup-ui))

modules/ui/doom-dashboard/config.el

@@ -263,7 +263,7 @@ controlled by `+doom-dashboard-pwd-policy'."
     (+doom-dashboard--center
      +doom-dashboard--width
      (format "Loaded %d packages in %d modules in %.02fs"
-             (length doom--package-load-path)
+             (length doom-package-load-path)
              (hash-table-size doom-modules)
              doom-init-time))
     'face 'font-lock-comment-face)