Rethink doom-initialize & use package-initalize

Doom saves a lot of startup time by avoiding package-initialize, because
it loads every packages' autoloads file, which is expensive.
Unfortunately, these autoloads files are necessary for some plugins with
abnormal file structures (such as ESS). Previously, I was content with
loading them myself, but these occurrences have become more frequent, so
it would be safest if I relied on package-initialize more.

So doom-initialize will now do so. However, it will cache the load-path
(and Info-directory-list, fixing the lost info buffers) on first run.
This makes byte-compiling Doom almost useless, but it makes startup just
as fast as Doom would be if you had.

This needs more testing.

core/core-packages.el

@@ -82,8 +82,11 @@ missing) and shouldn't be deleted.")
 (defvar doom-site-load-path load-path
   "The starting load-path, before it is altered by `doom-initialize'.")
 
-(defvar doom-package-load-path ()
+(defvar doom-autoload-file (concat doom-local-dir "autoloads.el")
-  "The load path of package libraries installed via ELPA and QUELPA.")
+  "Where `doom//reload-autoloads' will generate its autoloads file.")
+
+(defvar doom-packages-file (concat doom-local-dir "packages.el")
+  "Where to cache `load-path' and `Info-directory-list'.")
 
 (defvar doom--refreshed-p nil)
 (defvar doom--current-module nil)
@@ -134,19 +137,23 @@ If you byte-compile core/core.el, this function will be avoided to speed up
 startup."
   ;; Called early during initialization; only use native (and cl-lib) functions!
   (when (or force-p (not doom-init-p))
-    ;; Speed things up with a `load-path' for only the bare essentials
+    (unless (load doom-autoload-file t t t)
-    (let ((load-path doom-site-load-path))
+      (unless noninteractive
+        (error "No autoloads file! Run make autoloads")))
+    (when (or (not (load doom-packages-file t t t))
+              force-p)
       ;; Ensure core folders exist, otherwise we get errors
       (dolist (dir (list doom-local-dir doom-etc-dir doom-cache-dir doom-packages-dir))
         (unless (file-directory-p dir)
           (make-directory dir t)))
-      ;; Ensure package.el is initialized; we use its state
+      ;; Ensure packages have been initialized
+      (require 'package)
       (setq package-activated-list nil)
-      (condition-case _ (package-initialize t)
+      (condition-case _ (package-initialize)
         ('error (package-refresh-contents)
                 (setq doom--refreshed-p t)
-                (package-initialize t)))
+                (package-initialize)))
-      ;; Ensure core packages are installed
+      ;; Ensure core packages are installed.
       (let ((core-packages (cl-remove-if #'package-installed-p doom-core-packages)))
         (when core-packages
           (message "Installing core packages")
@@ -159,20 +166,13 @@ startup."
                 (message "✓ Installed %s" package)
               (error "✕ Couldn't install %s" package)))
           (message "Installing core packages...done")))
-      (setq doom-init-p t))))
+      (with-temp-buffer
-
+        (cl-pushnew doom-core-dir load-path :test #'string=)
-(defun doom-initialize-load-path (&optional force-p)
+        (prin1 `(setq load-path ',load-path
-  "Populates `load-path', if it hasn't already been. If FORCE-P is non-nil, do
+                      Info-directory-list ',Info-directory-list)
-it anyway."
+               (current-buffer))
-  (when (or force-p (not doom-package-load-path))
+        (write-file doom-packages-file)))
-    ;; We could let `package-initialize' fill `load-path', but it does more than
+    (setq doom-init-p t)))
-    ;; that alone (like load autoload files). If you want something prematurely
-    ;; optimizated right, ya gotta do it yourself.
-    ;;
-    ;; Also, in some edge cases involving package initialization during a
-    ;; non-interactive session, `package-initialize' fails to fill `load-path'.
-    (setq doom-package-load-path (directory-files package-user-dir t "^[^.]" t)
-          load-path (append doom-package-load-path doom-site-load-path))))
 
 (defun doom-initialize-autoloads ()
   "Ensures that `doom-autoload-file' exists and is loaded. Otherwise run
@@ -188,7 +188,7 @@ already. Also runs every enabled module's init.el.
 If FORCE-P is non-nil, do it even if they are.
 
 This aggressively reloads core autoload files."
-  (doom-initialize-load-path force-p)
+  (doom-initialize force-p)
   (with-temp-buffer ; prevent buffer-local settings from propagating
     (cl-flet
         ((_load
@@ -378,7 +378,8 @@ the lookup is relative to `load-file-name', `byte-compile-current-file' or
 `buffer-file-name' (in that order).
 
 If NOERROR is non-nil, don't throw an error if the file doesn't exist."
-  (cl-assert (symbolp filesym) t)
+  (or (symbolp filesym)
+      (signal 'wrong-type-argument (list 'symbolp filesym)))
   (let ((path (or path
                   (and load-file-name (file-name-directory load-file-name))
                   (and (bound-and-true-p byte-compile-current-file)
@@ -531,15 +532,14 @@ an Emacs session is running.
 This isn't necessary if you use Doom's package management commands because they
 call `doom//reload-load-path' remotely (through emacsclient)."
   (interactive)
-  (byte-recompile-file (expand-file-name "core.el" doom-core-dir) t)
   (cond ((and noninteractive (not (daemonp)))
          (require 'server)
          (when (server-running-p)
            (message "Reloading active Emacs session...")
            (server-eval-at server-name '(doom//reload-load-path))))
         ((let ((noninteractive t))
-           (doom-initialize-load-path t)
+           (doom-initialize t)
-           (message "%d packages reloaded" (length doom-package-load-path))
+           (message "%d packages reloaded" (length package-alist))
            (run-hooks 'doom-reload-hook)))))
 
 (defun doom//reload-autoloads ()
@@ -560,7 +560,7 @@ This should be run whenever init.el or an autoload file is modified. Running
       ;; state. `doom-initialize-packages' will have side effects otherwise.
       (progn
         (doom-packages--async-run 'doom//reload-autoloads)
-        (load doom-autoload-file t))
+        (load doom-autoload-file t nil t))
     (doom-initialize-packages t)
     (let ((targets
            (file-expand-wildcards
@@ -600,8 +600,8 @@ This should be run whenever init.el or an autoload file is modified. Running
                   (while (re-search-forward "^\\s-*(" nil t)
                     (unless (or (nth 4 (syntax-ppss))
                                 (nth 3 (syntax-ppss)))
-                      ;; Replace autoload paths with absolute paths for fastest
+                      ;; Replace autoload paths with absolute paths for faster
-                      ;; resolution during load
+                      ;; resolution during load and simpler `load-path'
                       (when (eq (sexp-at-point) 'autoload)
                         (save-excursion
                           (forward-sexp 2)

core/core.el

@@ -56,9 +56,6 @@ Use this for files that change often, like cache files.")
 (defvar doom-packages-dir (concat doom-local-dir "packages/")
   "Where package.el and quelpa plugins (and their caches) are stored.")
 
-(defvar doom-autoload-file (concat doom-local-dir "autoloads.el")
-  "Where `doom//reload-autoloads' will generate its autoloads file.")
-
 
 ;;;
 ;; UTF-8 as the default coding system
@@ -149,20 +146,10 @@ ability to invoke the debugger in debug mode."
           gc-cons-percentage 0.6
           file-name-handler-alist nil))
 
-  (load (concat doom-core-dir "core-packages") nil t)
+  (require 'core-packages (concat doom-core-dir "core-packages"))
-  (setq load-path (eval-when-compile (doom-initialize t)
+  (doom-initialize noninteractive)
-                                     (doom-initialize-load-path t))
-        doom-package-load-path (eval-when-compile doom-package-load-path))
-
   (load! core-lib)
   (load! core-os) ; consistent behavior across OSes
-  (condition-case-unless-debug ex
-      (require 'autoloads doom-autoload-file t)
-    ('error
-     (delete-file doom-autoload-file)
-     (lwarn 'doom-autoloads :warning
-            "Error in autoloads.el -> %s" ex)))
-
   (unless noninteractive
     (load! core-ui)         ; draw me like one of your French editors
     (load! core-editor)     ; baseline configuration for text editing