Heavy refactor of package management; new parsing doom-read-packages

core/autoload/packages.el

@@ -6,25 +6,24 @@
 ;;;###autoload
 (defun doom-refresh-packages ()
   "Refresh ELPA packages."
+  (doom-initialize)
   (when (or (not doom-packages-last-refresh)
             (> (nth 1 (time-since doom-packages-last-refresh)) 3600))
-    (doom-initialize)
     (package-refresh-contents)
     (setq doom-packages-last-refresh (current-time))))
 
 ;;;###autoload
-(defun doom-package-elpa-p (name)
+(defun doom-package-backend (name)
-  "Returns non-nil if NAME was a package installed with elpa."
+  "Get which backend the package NAME was installed with. Can either be elpa,
+quelpa or nil (if not installed)."
   (doom-initialize)
-  (and (assq name package-alist)
-       (not (doom-package-quelpa-p name))))
-
-;;;###autoload
-(defun doom-package-quelpa-p (name)
-  "Returns non-nil if NAME was a package installed with quelpa."
   (unless (quelpa-setup-p)
     (error "Could not initialize quelpa"))
-  (assq name quelpa-cache))
+  (cond ((or (assq name quelpa-cache)
+             (plist-get (cdr (assq name doom-packages)) :recipe))
+         'quelpa)
+        ((assq name package-alist)
+         'elpa)))
 
 ;;;###autoload
 (defun doom-package-outdated-p (name)
@@ -32,23 +31,24 @@
 list, whose car is NAME, and cdr the current version list and latest version
 list of the package."
   (doom-refresh-packages)
-  (package-read-all-archive-contents)
+  (let ((pkg (or (assq name package-alist)
-  (when (assq name package-alist)
+                 (assq name package--builtins))))
-    (let* ((old-version
+    (when pkg
-            (package-desc-version (cadr (or (assq name package-alist)
+      (let* ((old-version (package-desc-version (cadr pkg)))
-                                            (assq name package--builtins)))))
              (new-version
-            (cond ((doom-package-quelpa-p name)
+              (pcase (doom-package-backend name)
+                ('quelpa
                  (let ((recipe (assq name quelpa-cache))
                        (dir (f-expand (symbol-name name) quelpa-build-dir))
                        (inhibit-message t))
-                     (or (quelpa-checkout recipe dir)
+                   (or (and (quelpa-setup-p) (quelpa-checkout recipe dir))
                        old-version)))
-
+                ('elpa
-                  ((doom-package-elpa-p name)
                  (package-desc-version (cadr (assq name package-archive-contents)))))))
-      (unless (version-list-<= new-version old-version)
+        (when (stringp new-version)
-        (cons name old-version new-version)))))
+          (setq new-version (version-to-list new-version)))
+        (when (version-list-< old-version new-version)
+          (list name old-version new-version))))))
 
 ;;;###autoload
 (defun doom-get-packages (&optional backend)
@@ -58,12 +58,14 @@ the quelpa recipe (if any).
 
 BACKEND can be 'quelpa or 'elpa, and will instruct this function to return only
 the packages relevant to that backend."
-  (doom-reload)
+  (doom-read-packages t)
   (unless (quelpa-setup-p)
     (error "Could not initialize quelpa"))
-  (--map (cons it (assq it quelpa-cache))
+  (-non-nil
-         (-intersection (package--find-non-dependencies)
+   (--map (or (assq it doom-packages)
-                        (append (mapcar 'car doom-packages) doom-protected-packages))))
+              (list (car (assq it package-alist))))
+          (append (mapcar 'car doom-packages)
+                  doom-protected-packages))))
 
 ;;;###autoload
 (defun doom-get-outdated-packages ()
@@ -76,16 +78,15 @@ be fed to `doom/packages-update'."
 (defun doom-get-orphaned-packages ()
   "Return a list of packages that are no longer needed or depended on. Can be
 fed to `doom/packages-delete'."
-  (doom-reload)
+  (doom-read-packages t)
-  (let ((package-selected-packages
+  (let ((package-selected-packages (append (mapcar 'car doom-packages) doom-protected-packages)))
-         (append (mapcar 'car doom-packages) doom-protected-packages)))
     (package--removable-packages)))
 
 ;;;###autoload
 (defun doom-get-packages-to-install ()
   "Return a list of packages that aren't installed, but need to be. Used by
 `doom/packages-install'."
-  (doom-reload)
+  (doom-read-packages t)
   (--remove (assq (car it) package-alist)
             (append doom-packages (-map 'list doom-protected-packages))))
 
@@ -106,15 +107,29 @@ fed to `doom/packages-delete'."
 ;; Main functions
 ;;
 
-(defun doom-install-package (name &optional recipe)
+(defun doom-install-package (name &optional plist)
   "Installs package NAME with optional quelpa RECIPE (see `quelpa-recipe' for an
 example; the package name can be omitted)."
   (doom-refresh-packages)
   (when (package-installed-p name)
-    (error "%s is already installed" name))
+    (error "%s is already installed, skipping" name))
-  (cond (recipe (quelpa (plist-get plist :recipe)))
+  (when (plist-get plist :disabled)
+    (error "%s is disabled, skipping" name))
+  (when (plist-get plist :load-path)
+    (error "%s has a local load-path, skipping" name))
+  (cond ((plist-get plist :recipe)
+         (let ((recipe (plist-get plist :recipe)))
+           (when (and recipe (= 0 (mod (length recipe) 2)))
+             (setq recipe (cons name recipe)))
+           (quelpa recipe)))
         (t (package-install name)))
-  (add-to-list 'doom-packages (cons name recipe))
+  (cl-pushnew (cons name plist) doom-packages :key 'car)
+  (when (plist-member plist :setup)
+    (let ((setup (plist-get plist :setup)))
+      (when (listp setup)
+        (setq setup (assq (doom-os) setup)))
+      (when setup
+        (async-shell-command setup))))
   (package-installed-p name))
 
 (defun doom-update-package (name)
@@ -125,18 +140,18 @@ appropriate."
     (error "%s isn't installed" name))
   (when (doom-package-outdated-p name)
     (let (quelpa-modified-p)
-      (cond ((doom-package-quelpa-p name)
+      (pcase (doom-package-backend name)
+        ('quelpa
          (let ((quelpa-upgrade-p t))
            (quelpa it)
            (setq quelpa-modified-p t)))
-            (t
+        ('elpa
          (let ((desc    (cadr (assq name package-alist)))
                (archive (cadr (assq name package-archive-contents))))
            (package-install-from-archive archive)
-               (delete-directory (package-desc-dir desc) t))
+           (delete-directory (package-desc-dir desc) t))))
-             (package-install name))))
       (when quelpa-modified-p
-      (quelpa-save-cache))
+        (quelpa-save-cache)))
     (version-list-=
      (package-desc-version (cadr (assq name package-alist)))
      (package-desc-version (cadr (assq name package-archive-contents))))))
@@ -165,13 +180,17 @@ appropriate."
            (message "No packages to install!"))
 
           ((not (y-or-n-p
-                 (format "%s packages will be installed:\n%s\n\nProceed?"
+                 (format "%s packages will be installed:\n\n%s\n\nProceed?"
                          (length packages)
-                         (mapconcat (lambda (pkg) (format "+ %s (%s)"
+                         (mapconcat (lambda (pkg)
-                                                     (symbol-name (car pkg))
+                                      (format "+ %s (%s)"
-                                                     (cond ((cdr pkg) "QUELPA")
+                                              (car pkg)
+                                              (cond ((plist-get (cdr pkg) :recipe) "QUELPA")
                                                     (t "ELPA"))))
-                                    packages "\n"))))
+                                    (--sort (string-lessp (symbol-name (car it))
+                                                          (symbol-name (car other)))
+                                            packages)
+                                    "\n"))))
            (message "Aborted!"))
 
           (t
@@ -180,10 +199,9 @@ appropriate."
            (dolist (pkg packages)
              (condition-case ex
                  (doom-message "%s %s (%s)"
-                               (let ((plist (cdr pkg)))
                                (if (doom-install-package (car pkg) (cdr pkg))
                                    "Installed"
-                                   "Failed to install"))
+                                 "Failed to install")
                                pkg
                                (cond ((cdr pkg) "QUELPA")
                                      (t "ELPA")))
@@ -201,27 +219,29 @@ appropriate."
            (message "Everything is up-to-date"))
 
           ((not (y-or-n-p
-                 (format "%s packages will be updated:\n%s\n\nProceed?"
+                 (format "%s packages will be updated:\n\n%s\n\nProceed?"
                          (length packages)
-                         (mapconcat (lambda (pkg) (format "%s: %s -> %s"
+                         (mapconcat
-                                                     (car pkg)
+                          (lambda (pkg) (format "+ %s %s -> %s\t%s"
+                                           (s-pad-right 20 " " (symbol-name (car pkg)))
                                            (car (cdr pkg))
-                                                     (cdr (cdr pkg))))
+                                           (car (cdr (cdr pkg)))))
                           (--sort (string-lessp (symbol-name (car it))
                                                 (symbol-name (car other)))
-                                            outdated-packages) ", "))))
+                                  packages)
+                          "\n"))))
            (message "Aborted!"))
 
           (t
            (dolist (pkg packages)
              (condition-case ex
                  (doom-message "%s %s"
-                               (if (doom-update-package pkg)
+                               (if (doom-update-package (car pkg))
                                    "Updated"
                                  "Failed to update")
-                               pkg)
+                               (car pkg))
                (error
-                (doom-message "Error installing %s: %s" pkg ex))))
+                (doom-message "Error installing %s: %s" (car pkg) ex))))
 
            (doom-message "Finished!")))))
 
@@ -234,7 +254,7 @@ appropriate."
            (message "No unused packages to remove"))
 
           ((not (y-or-n-p
-                 (format "%s packages will be deleted:\n%s\n\nProceed?"
+                 (format "%s packages will be deleted:\n\n%s\n\nProceed?"
                          (length packages)
                          (mapconcat 'symbol-name (-sort 'string-lessp packages) ", "))))
            (message "Aborted!"))
@@ -253,10 +273,10 @@ appropriate."
            (doom-message "Finished!")))))
 
 ;;;###autoload
-(defalias 'doom/package-install 'package-install)
+(defalias 'doom/install-package 'package-install)
 
 ;;;###autoload
-(defun doom/package-delete (&optional package)
+(defun doom/delete-package (&optional package)
   (interactive
    (list (completing-read "Delete package: " (doom-get-packages))))
   (if (package-installed-p package)
@@ -268,7 +288,7 @@ appropriate."
     (message "%s isn't installed" package)))
 
 ;;;###autoload
-(defun doom/package-update (&optional package)
+(defun doom/update-package (&optional package)
   (interactive
    (list (completing-read "Update package: " (doom-get-packages))))
   (if (doom-package-outdated-p package)

core/core-editor.el

@@ -184,6 +184,8 @@
 (package! smart-forward
   :commands (smart-up smart-down smart-backward smart-forward))
 
+(package! smex :commands smex)
+
 (package! swiper :commands (swiper swiper-all))
 
 (package! wgrep

core/core-lib.el

@@ -7,6 +7,21 @@
 (require 'f)
 (require 's)
 
+(package! anaphora
+  :commands (awhen aif acond awhile))
+
+(package! async
+  :commands (async-start
+             async-start-process
+             async-byte-recompile-directory))
+
+(package! persistent-soft
+  :preface (defvar pcache-directory (concat doom-cache-dir "pcache/"))
+  :commands (persistent-soft-exists-p
+             persistent-soft-fetch
+             persistent-soft-flush
+             persistent-soft-store))
+
 
 ;;
 ;; Library
@@ -68,6 +83,7 @@ Examples:
 (defmacro associate! (mode &rest plist)
   "Associate a major or minor mode to certain patterns and project files."
   (declare (indent 1))
+  (unless noninteractive
     (let* ((minor (plist-get plist :minor))
            (in    (plist-get plist :in))
            (match (plist-get plist :match))
@@ -96,7 +112,7 @@ Examples:
              `(add-to-list ',(if minor 'doom-auto-minor-mode-alist 'auto-mode-alist)
                            (cons ,match ',mode)))
             (t (user-error "associate! invalid rules for mode [%s] (in %s) (match %s) (files %s)"
-                         mode in match files)))))
+                           mode in match files))))))
 
 ;; Register keywords for proper indentation (see `map!')
 (put ':prefix       'lisp-indent-function 'defun)
@@ -159,6 +175,7 @@ Example
           (:when IS-MAC
            :n \"M-s\" 'some-fn
            :i \"M-o\" (lambda (interactive) (message \"Hi\"))))"
+  (unless noninteractive
     (let ((keymaps (if (boundp 'keymaps) keymaps))
           (prefix  (if (boundp 'prefix) prefix))
           (state-map '(("n" . normal)
@@ -244,12 +261,22 @@ Example
                   local nil)))
 
          (t (user-error "Invalid key %s" key))))
-    (macroexp-progn (reverse forms))))
+      (macroexp-progn (reverse forms)))))
 
-(when (or noninteractive doom-dont-load-p)
+(defun doom-os ()
-  (defmacro add-hook! (&rest _))
+  "Returns the OS: arch, debian, macos, general linux, cygwin or windows."
-  (defmacro associate! (&rest _))
+  (let ((gnu-linux-p (eq system-type 'gnu/linux)))
-  (defmacro map! (&rest _)))
+    (cond ((and gnu-linux-p (f-exists-p "/etc/arch-release"))
+           'arch)
+          ((and gnu-linux-p (f-exists-p "/etc/debian_version"))
+           'debian)
+          (gnu-linux-p
+           'linux)
+          ((eq system-type 'darwin)
+           'macos)
+          ((memq system-type '(windows-nt cygwin))
+           'windows)
+          (t (error "Unknown OS: %s" system-type)))))
 
 (provide 'core-lib)
 ;;; core-lib.el ends here

core/core-packages.el

@@ -32,18 +32,6 @@ submodule symbol, e.g. 'evil.")
   "Non-nil if doom's package system has been initialized or not. It may not be
 if you have byte-compiled your configuration (as intended).")
 
-(defvar doom-dont-load-p nil
-  "If non-nil, run DOOM emacs in declarative mode, meaning: don't actually load
-anything, just track what should be loaded. Useful for scanning packages and
-loaded modules.")
-
-(defvar doom-reloading-p nil
-  "If non-nil, DOOM is reloading itself. Use this to determine to prevent
-infinite recursion.")
-
-(defvar doom-prefer-el-p noninteractive
-  "If non-nil, load uncompiled .el config files.")
-
 (defvar doom--base-load-path (append (list doom-core-dir
                                            doom-modules-dir)
                                      load-path)
@@ -83,14 +71,13 @@ infinite recursion.")
 byte-compilation."
   (let (mode)
     (dolist (p packages)
-      (cond ((string-prefix-p ":" (symbol-name p))
+      (cond ((keywordp p)
              (setq mode p))
             ((not mode)
              (error "No namespace specified on `doom!' for %s" p))
             (t
              (setq doom-enabled-modules (append doom-enabled-modules (list (cons mode p))))))))
-  `(unless doom-dont-load-p
+  `(let (file-name-handler-alist)
-     (let (file-name-handler-alist)
      ,@(mapcar (lambda (pkg) `(load! ,(car pkg) ,(cdr pkg)))
                doom-enabled-modules)
 
@@ -103,7 +90,7 @@ byte-compilation."
      (advice-add 'display-startup-echo-area-message :override 'ignore)
      (message "Loaded %s packages in %s"
               (- (length load-path) (length doom--base-load-path))
-                (emacs-init-time)))))
+              (emacs-init-time))))
 
 (defun doom-initialize (&optional force-p)
   "Initialize installed packages (using package.el) and ensure the core packages
@@ -142,25 +129,6 @@ avoided to speed up startup."
           '(:ensure :pin))
     (setq doom-init-p t)))
 
-(defun doom-reload ()
-  "Rereads the Emacs config, reloading `doom-packages' and
-`doom-enabled-modules'."
-  (unless doom-reloading-p
-    (doom-initialize)
-    (let ((doom-prefer-el-p t)
-          (doom-dont-load-p t)
-          (doom-reloading-p t)
-          noninteractive)
-      (setq doom-enabled-modules nil
-            doom-packages nil)
-      (let ((load-fn (lambda (file) (load file :noerror (not doom-debug-mode) :nosuffix))))
-        (mapc load-fn
-              (list (f-expand "init.el" doom-emacs-dir)
-                    (f-expand "core.el" doom-core-dir)))
-        (mapc load-fn
-              (--map (doom-module-path (car it) (cdr it) "packages.el")
-                     doom-enabled-modules))))))
-
 
 ;;
 ;; Macros
@@ -170,6 +138,19 @@ avoided to speed up startup."
 (defvar __FILE__ nil "The full path of the currently loaded file (set by `load!')")
 (defvar __PACKAGE__ nil "The name of the current package.")
 
+(defun __DIR__ ()
+  (or __DIR__
+      (and load-file-name (f-dirname load-file-name))
+      (and buffer-file-name (f-dirname buffer-file-name))
+      default-directory
+      (error "__DIR__ is unset")))
+
+(defun __FILE__ ()
+  (or __FILE__
+      load-file-name
+      buffer-file-name
+      (error "__FILE__ is unset")))
+
 (defmacro use-package! (name &rest plist)
   "A `use-package' wrapper. It exists so configs can adhere to the naming
 conventions of DOOM emacs, as well as let-bind `__PACKAGE__' for the containing
@@ -180,46 +161,31 @@ packages are deferred by default."
      (use-package ,name ,@plist)))
 
 (defmacro package! (name &rest plist)
-  "Wraps around `use-package' to declare a deferred package (unless otherwise
+  "Declares a package. This does not actually load nor install them explicitly.
-indicated), takes the same arguments, but adds a few custom properties:
+
+If used in `doom-core-dir', this is a wrapper for `use-package!' (all packages
+are deferred by default), and takes the same arguments as `use-package'.
+
+If used outside of `doom-core-dir', this macro is purely declarative and doesn't
+call `use-package!'. These calls are parsed by package management functions,
+such as `doom-read-packages'.
+
+Adds a few custom properties in either case:
 
  :recipe RECIPE        Takes a MELPA-style recipe (see `quelpa-recipe' for an
                        example); for packages to be installed from external
                        sources.
  :pin ARCHIVE-NAME     Instructs ELPA to only look for this package in
                        ARCHIVE-NAME. e.g. \"org\".
- :needs FEATURE        Don't install this package if FEATURE isn't available.
+ :needs FEATURE        Don't install this package if FEATURE isn't available. Can be a
-
+                       (:module . submodule) cons pair.
-Also binds `__PACKAGE__` for PLIST forms to optionally use."
+ :setup CMD-OR-PCASE   A command to run after install. Can be a pcase list, whose
+                       car's are symbols of OSes that `doom-os' returns, and whose
+                       cdr's are string shell commands."
   (declare (indent defun))
-  (let ((recipe (cadr (memq :recipe plist)))
+  (mapc (lambda (key) (setq plist (use-package-plist-delete plist key)))
-        (pin    (cadr (memq :pin plist)))
+        '(:recipe :pin :setup :needs))
-        (lpath  (cadr (memq :load-path plist)))
+  `(use-package! ,name ,@plist))
-        (dep    (cadr (memq :needs plist))))
-    (when (or (not dep)
-              (or (featurep dep)
-                  (package-installed-p dep)))
-      (when (and recipe (= 0 (mod (length recipe) 2)))
-        (push name recipe))
-      (if (not lpath)
-          (cl-pushnew (cons name recipe) doom-packages :key 'car)
-        (cl-pushnew lpath doom--base-load-path)
-        (setq recipe nil
-              pin nil))
-      (when pin
-        (cl-pushnew (cons package (plist-get plist :pin)) package-pinned-packages :key 'car))
-      (setq plist (use-package-plist-delete plist :recipe))
-      (setq plist (use-package-plist-delete plist :pin))
-      (unless doom-dont-load-p
-        `(use-package! ,name ,@plist)))))
-
-(defmacro require! (feature)
-  "Like `require', but will prefer uncompiled files if `doom-prefer-el-p' is
-non-nil or this is a noninteractive session."
-  (let ((prefer-el-p (or doom-prefer-el-p noninteractive)))
-    `(require ',feature
-              ,(locate-file (concat (symbol-name feature) (if prefer-el-p ".el"))
-                            load-path))))
 
 (defmacro load! (module &optional submodule file)
   "Load a module from `doom-modules-dir' when both MODULE and SUBMODULE is
@@ -227,9 +193,6 @@ provided (both symbols). If FILE is non-nil, append it to the resulting path. If
 SUBMODULE is nil, MODULE is loaded relative to the current file (see `__DIR__').
 When SUBMODULE is nil, FILE isn't used.
 
-Will prefer uncompiled elisp sources if `doom-prefer-el-p' is non-nil or this is
-an noninteractive session.
-
 Examples:
  (load! :lang emacs-lisp)
 
@@ -238,8 +201,7 @@ Examples:
  (load! +local-module)
 
   Loads +local-module.el relative to `__DIR__' or `doom-core-dir'."
-  (let ((prefer-el-p (or doom-prefer-el-p noninteractive))
+  (let (path file)
-        path file)
     (cond ((null submodule)
            (setq path __DIR__
                  file (concat (symbol-name module) ".el")))
@@ -254,8 +216,7 @@ Examples:
           file (concat path file))
     `(let ((__FILE__ ,file)
            (__DIR__  ,path))
-       (load ,(if doom-prefer-el-p file (f-no-ext file))
+       (load ,(f-no-ext file) nil (not doom-debug-mode)))))
-             nil (not doom-debug-mode) ,doom-prefer-el-p))))
 
 (defun doom-module-path (module submodule &optional file)
   "Get the full path to a module: e.g. :lang emacs-lisp maps to
@@ -276,8 +237,8 @@ Examples:
 ;;
 
 (defun doom/reload ()
-  "Reload `load-path' by scanning all packages. Run this if you ran make update
+  "Reload `load-path' by reinitializing package.el. Run this if you ran update
-or make clean outside of Emacs."
+or delete packages from outside of Emacs."
   (interactive)
   (doom-initialize t)
   (message "Reloaded %s packages" (length package-alist)))
@@ -291,8 +252,7 @@ In modules, checks for modules/*/autoload.el and modules/*/autoload/*.el.
 Rerun this whenever you modify your init.el (or use `make autoloads` from the
 command line)."
   (interactive)
-  (unless doom-reloading-p
+  (doom-read-packages nil t)
-    (doom-reload)
   (let ((generated-autoload-file (concat doom-local-dir "autoloads.el"))
         (autoload-files
          (append (-flatten (mapcar (lambda (dir)
@@ -302,7 +262,8 @@ command line)."
                                               (f-glob "*.el" auto-dir))
                                              ((f-exists-p auto-file)
                                               auto-file))))
-                                     (--map (doom-module-path (car it) (cdr it)) doom-enabled-modules)))
+                                   (--map (doom-module-path (car it) (cdr it))
+                                          doom-enabled-modules)))
                  (f-glob "autoload/*.el" doom-core-dir))))
     (when (f-exists-p generated-autoload-file)
       (f-delete generated-autoload-file)
@@ -313,7 +274,7 @@ command line)."
     (with-current-buffer (get-file-buffer generated-autoload-file)
       (save-buffer)
       (eval-buffer))
-      (message "Done!"))))
+    (message "Done!")))
 
 (defun doom/byte-compile (&optional comprehensive-p)
   "Byte (re)compile the important files in your emacs configuration (i.e.
@@ -323,7 +284,7 @@ a lot from this.
 If COMPREHENSIVE-P is non-nil, then compile modules/*/*/*.el (except for
 packages.el files) -- this will likely take a long time."
   (interactive)
-  (doom-reload)
+  (doom-read-packages)
   (let ((targets (append
                   (list (f-expand "init.el" doom-emacs-dir)
                         (f-expand "core.el" doom-core-dir))
@@ -338,18 +299,98 @@ packages.el files) -- this will likely take a long time."
                           doom-enabled-modules))))
         (n 0)
         results)
-    (mapc (lambda (file)
+    (dolist (file targets)
       (push (cons (f-relative file doom-emacs-dir)
                   (when (byte-recompile-file file nil 0)
                     (setq n (1+ n))
                     t))
             results))
-          targets)
     (when noninteractive
       (when targets (message "\n"))
-      (message "Compiling %s files:\n%s" n
+      (message "Compiled %s files:\n%s" n
                (mapconcat (lambda (file) (concat "+ " (if (cdr file) "SUCCESS" "FAIL") ": " (car file)))
                           (reverse results) "\n")))))
 
+
+;;
+;; Package parsing
+;;
+
+(defun doom--parse-forms (sym forms)
+  (let ((result-forms (and (boundp 'result-forms) result-forms)))
+    (dolist (form forms)
+      (cond ((eq (car-safe form) sym)
+             (push (cdr-safe form) result-forms))
+
+            ((and (listp form)
+                  (not (-cons-pair? form)))
+             (setq result-forms (doom--parse-forms sym form)))))
+    result-forms))
+
+(defun doom--parse-file-forms (sym file)
+  (declare (indent defun))
+  (unless (f-exists-p file)
+    (error "%s does not exist" file))
+  (unless (symbolp sym)
+    (error "%s is not a valid symbol" sym))
+  (let (forms)
+    (with-temp-buffer
+      (insert "(setq forms '(\n")
+      (insert-file-contents file)
+      (goto-char (point-max))
+      (insert "\n))")
+      (eval-buffer))
+    (doom--parse-forms sym forms)))
+
+(defun doom--strip-property (plist property)
+  (let (forms)
+    (while (and plist (not (eq (car plist) property)))
+      (setq forms (append forms (list (pop plist)))))
+    (pop plist)
+    (while (and plist (not (keywordp (car plist))))
+      (pop plist))
+    (when plist
+      (setq forms (append forms plist)))
+    forms))
+
+;;;###autoload
+(defun doom-read-packages (&optional force-p nopackages)
+  "Parses your Emacs config to keep track of packages declared with `package!'
+in `doom-packages' and enabled modules in `doom-enabled-modules'."
+  (doom-initialize)
+  (when (or force-p (not doom-enabled-modules) (not doom-packages))
+    (setq doom-enabled-modules
+          (let (paths mode enabled-modules)
+            (--each (doom--parse-file-forms 'doom! (f-expand "init.el" doom-emacs-dir))
+              (dolist (module it)
+                (cond ((keywordp module)
+                       (setq mode module))
+                      ((not mode)
+                       (error "Malformed doom! call: no namespace for %s" module))
+                      (t
+                       (push (cons mode module) enabled-modules)))))
+            enabled-modules))
+
+    (unless nopackages
+      (setq package-pinned-packages nil
+            doom-packages nil)
+      (mapc (lambda (pkg) (cl-pushnew pkg doom-packages :key 'car))
+            (mapcar (lambda (args)
+                      (mapc (lambda (keyword) (setq args (doom--strip-property args keyword)))
+                            '(:preface :ensure :requires :no-require :bind :bind* :bind-keymap
+                              :bind-keymap* :interpreter :mode :commands :defines :functions
+                              :defer :init :after :demand :config :diminish :delight))
+                      args)
+                    (--sort (string-greaterp (symbol-name (car it))
+                                             (symbol-name (car other)))
+                            (-flatten-n
+                             1 (mapcar (lambda (file)
+                                         (when (f-exists-p file)
+                                           (doom--parse-file-forms 'package! file)))
+                                       (append (f-glob "core*.el" doom-core-dir)
+                                               (--map (doom-module-path (car it) (cdr it) "packages.el")
+                                                      doom-enabled-modules)))))))
+      t)))
+
 (provide 'core-packages)
 ;;; core-packages.el ends here

core/core.el

@@ -130,30 +130,12 @@ enable multiple minor modes for the same regexp.")
   (require 'core-lib)
   (require 'autoloads (concat doom-local-dir "autoloads.el") t)
   (unless noninteractive
-    (package! anaphora
+    (require 'core-set)         ; a centralized config system; provides `set!'
-      :commands (awhen aif acond awhile))
+    (require 'core-states)      ; TODO
-
+    (require 'core-ui)          ; draw me like one of your French editors
-    (package! async
+    (require 'core-popups)      ; taming sudden yet inevitable windows
-      :commands (async-start
+    (require 'core-editor)      ; baseline configuration for text editing
-                 async-start-process
+    (require 'core-projects)    ; making Emacs project-aware
-                 async-byte-recompile-directory))
-
-    (package! persistent-soft
-      :preface (defvar pcache-directory (concat doom-cache-dir "pcache/"))
-      :commands (persistent-soft-exists-p
-                 persistent-soft-fetch
-                 persistent-soft-flush
-                 persistent-soft-store))
-
-    (package! smex :commands smex)
-
-    ;;
-    (require! core-set)         ; a centralized config system; provides `set!'
-    (require! core-states)      ; TODO
-    (require! core-ui)          ; draw me like one of your French editors
-    (require! core-popups)      ; taming sudden yet inevitable windows
-    (require! core-editor)      ; baseline configuration for text editing
-    (require! core-projects)    ; making Emacs project-aware
 
     ;; We check last as a promise that the core files won't use autoloaded
     ;; functions. If they do, it shouldn't be autoloaded!