diff --git a/lisp/doom-packages.el b/lisp/doom-packages.el
index e69bdd9ec..99ec8b435 100644
--- a/lisp/doom-packages.el
+++ b/lisp/doom-packages.el
@@ -107,6 +107,23 @@ uses a straight or package.el command directly).")
           (doom-package-pinned-list)))
 
 
+;;
+;;; native-comp
+
+(when NATIVECOMP
+  (after! comp
+    ;; HACK Disable native-compilation for some troublesome packages
+    (mapc (doom-partial #'add-to-list 'native-comp-deferred-compilation-deny-list)
+          (let ((local-dir-re (concat "\\`" (regexp-quote doom-local-dir))))
+            (list (concat local-dir-re ".*/emacs-jupyter.*\\.el\\'")
+                  (concat local-dir-re ".*/evil-collection-vterm\\.el\\'")
+                  (concat local-dir-re ".*/with-editor\\.el\\'")))))
+
+  ;; Remove unwanted $EMACSDIR/eln-cache/ directory.
+  (cl-callf2 delete (file-name-concat doom-emacs-dir "eln-cache/")
+             native-comp-eln-load-path))
+
+
 ;;
 ;;; Bootstrappers
 
diff --git a/lisp/doom.el b/lisp/doom.el
index fd8efba3b..807ad09fa 100644
--- a/lisp/doom.el
+++ b/lisp/doom.el
@@ -294,15 +294,7 @@ users).")
 (when NATIVECOMP
   ;; Don't store eln files in ~/.emacs.d/eln-cache (where they can easily be
   ;; deleted by 'doom upgrade').
-  (add-to-list 'native-comp-eln-load-path (concat doom-cache-dir "eln/"))
-
-  (with-eval-after-load 'comp
-    ;; HACK Disable native-compilation for some troublesome packages
-    (mapc (apply-partially #'add-to-list 'native-comp-deferred-compilation-deny-list)
-          (let ((local-dir-re (concat "\\`" (regexp-quote doom-local-dir))))
-            (list (concat local-dir-re ".*/emacs-jupyter.*\\.el\\'")
-                  (concat local-dir-re ".*/evil-collection-vterm\\.el\\'")
-                  (concat local-dir-re ".*/with-editor\\.el\\'"))))))
+  (add-to-list 'native-comp-eln-load-path (file-name-concat doom-cache-dir "eln/")))
 
 
 ;;