diff --git a/core/core-dispatcher.el b/core/core-dispatcher.el
index 0ae3c0e27..1f45166fe 100644
--- a/core/core-dispatcher.el
+++ b/core/core-dispatcher.el
@@ -711,7 +711,8 @@ If RECOMPILE-P is non-nil, only recompile out-of-date files."
           (message "Aborting.")
           (cl-return-from 'byte-compile)))
       (unless recompile-p
-        (doom-clean-byte-compiled-files)
+        (when (or (null modules) (equal modules '(":core")))
+          (doom-clean-byte-compiled-files))
         (doom-reload-autoloads))
       (let (doom-emacs-changed-p
             noninteractive)