diff --git a/core/autoload/line-numbers.el b/core/autoload/line-numbers.el
index 1fa32d7a6..6ea35584d 100644
--- a/core/autoload/line-numbers.el
+++ b/core/autoload/line-numbers.el
@@ -1,5 +1,5 @@
 ;;; core/autoload/line-numbers.el -*- lexical-binding: t; -*-
-;;;###if (not (locate-library "display-line-numbers"))
+;;;###if (not EMACS26+)
 
 ;; This was lifted out of the display-line-numbers library in Emacs 26.1 and
 ;; modified to use nlinum for Emacs 25.x users. It should be removed should
@@ -26,8 +26,6 @@ relative number of a line.
 Lisp programs can disable display of a line number of a particular
 buffer line by putting the display-line-numbers-disable text property
 or overlay property on the first visible character of that line.")
-;;;###autoload
-(put 'display-line-numbers 'nlinum t)
 
 ;;;###autoload
 (defcustom display-line-numbers-type t
diff --git a/core/autoload/ui.el b/core/autoload/ui.el
index 8a0ebaee5..8866cf76e 100644
--- a/core/autoload/ui.el
+++ b/core/autoload/ui.el
@@ -62,25 +62,24 @@ visual-line-mode is on, this skips relative and uses visual instead.
 See `display-line-numbers' for what these values mean."
   (interactive)
   (defvar doom--line-number-style display-line-numbers-type)
-  (let ((nlinum-p (get 'display-line-numbers 'nlinum)))
-    (let* ((styles `(t ,(if (and (not nlinum-p) visual-line-mode) 'visual 'relative) nil))
-           (order (cons display-line-numbers-type (remq display-line-numbers-type styles)))
-           (queue (memq doom--line-number-style order))
-           (next (if (= (length queue) 1)
-                     (car order)
-                   (car (cdr queue)))))
-      (setq doom--line-number-style next)
-      (if (get 'display-line-numbers 'nlinum)
-          (pcase next
-            (`t (nlinum-relative-off) (nlinum-mode +1))
-            (`relative (nlinum-relative-on))
-            (`nil (nlinum-mode -1)))
-        (setq display-line-numbers next))
-      (message "Switched to %s line numbers"
-               (pcase next
-                 (`t "normal")
-                 (`nil "disabled")
-                 (x (symbol-name next)))))))
+  (let* ((styles `(t ,(if (and EMACS26+ visual-line-mode) 'visual 'relative) nil))
+         (order (cons display-line-numbers-type (remq display-line-numbers-type styles)))
+         (queue (memq doom--line-number-style order))
+         (next (if (= (length queue) 1)
+                   (car order)
+                 (car (cdr queue)))))
+    (setq doom--line-number-style next)
+    (if EMACS26+
+        (setq display-line-numbers next)
+      (pcase next
+        (`t (nlinum-relative-off) (nlinum-mode +1))
+        (`relative (nlinum-relative-on))
+        (`nil (nlinum-mode -1))))
+    (message "Switched to %s line numbers"
+             (pcase next
+               (`t "normal")
+               (`nil "disabled")
+               (_ (symbol-name next))))))
 
 ;;;###autoload
 (defun doom/reload-theme ()
diff --git a/core/core-ui.el b/core/core-ui.el
index a53c12e08..b9972e780 100644
--- a/core/core-ui.el
+++ b/core/core-ui.el
@@ -121,8 +121,7 @@ shorter major mode name in the mode-line. See `doom|set-mode-name'.")
 (setq visual-fill-column-center-text t
       visual-fill-column-width
       ;; take Emacs 26 line numbers into account
-      (+ (if (boundp 'display-line-numbers) 6 0)
-         fill-column))
+      (+ (if EMACS26+ 6 0) fill-column))
 
 (defun doom*hide-undefined-which-key-binds (bindings)
   (cl-loop for bind in bindings
@@ -242,18 +241,20 @@ shorter major mode name in the mode-line. See `doom|set-mode-name'.")
 
 ;;
 ;; Line numbers
-;;
+
+;; line numbers in most modes
+(add-hook! (prog-mode text-mode conf-mode) #'display-line-numbers-mode)
 
 ;; Emacs 26+ has native line number support, and will ignore nlinum. This is for
 ;; Emacs 25 users:
 (defun doom|enable-line-numbers ()  (display-line-numbers-mode +1))
 (defun doom|disable-line-numbers () (display-line-numbers-mode -1))
 
-;; Line number column. A faster (or equivalent, in the worst case) line number
-;; plugin than `linum-mode'.
 (def-package! nlinum
-  :when (get 'display-line-numbers 'nlinum)
-  :commands nlinum-mode
+  ;; Line number column. A faster (or equivalent, in the worst case) line number
+  ;; plugin than `linum-mode'.
+  :unless EMACS26+
+  :defer t
   :init
   (defvar doom-line-number-lpad 4
     "How much padding to place before line numbers.")
@@ -298,9 +299,9 @@ character that looks like a space that `whitespace-mode' won't affect.")
                                   (format-mode-line "%l")))))
   (add-hook 'nlinum-mode-hook #'doom|init-nlinum-width))
 
-;; Fixes disappearing line numbers in nlinum and other quirks
 (def-package! nlinum-hl
-  :when (get 'display-line-numbers 'nlinum)
+  ;; Fixes disappearing line numbers in nlinum and other quirks
+  :unless EMACS26+
   :after nlinum
   :config
   ;; With `markdown-fontify-code-blocks-natively' enabled in `markdown-mode',
@@ -315,8 +316,8 @@ character that looks like a space that `whitespace-mode' won't affect.")
   (advice-add #'set-frame-font :after #'nlinum-hl-flush-all-windows))
 
 (def-package! nlinum-relative
-  :when (get 'display-line-numbers 'nlinum)
-  :commands (nlinum-relative-mode nlinum-relative-on nlinum-relative-off)
+  :unless EMACS26+
+  :defer t
   :config
   (setq nlinum-format " %d ")
   (add-hook 'evil-mode #'nlinum-relative-setup-evil))
@@ -399,10 +400,6 @@ frame's window-system, the theme will be reloaded.")
 
 ;; a good indicator that Emacs isn't frozen
 (add-hook 'doom-init-ui-hook #'blink-cursor-mode)
-
-;; line numbers in most modes
-(add-hook! (prog-mode text-mode conf-mode) #'display-line-numbers-mode)
-
 ;; Make `next-buffer', `other-buffer', etc. ignore unreal buffers.
 (add-to-list 'default-frame-alist (cons 'buffer-predicate #'doom-buffer-frame-predicate))