diff --git a/core/core-ui.el b/core/core-ui.el
index ea9de1970..016bf2ca1 100644
--- a/core/core-ui.el
+++ b/core/core-ui.el
@@ -164,9 +164,6 @@ or if the current buffer is read-only or not file-visiting."
       mouse-wheel-scroll-amount '(2 ((shift) . hscroll))
       mouse-wheel-scroll-amount-horizontal 2)
 
-;; Remove hscroll-margin in shells, otherwise it causes jumpiness
-(setq-hook! '(eshell-mode-hook term-mode-hook) hscroll-margin 0)
-
 
 ;;
 ;;; Cursor
diff --git a/modules/term/eshell/config.el b/modules/term/eshell/config.el
index 95dff3460..ac95ddbd1 100644
--- a/modules/term/eshell/config.el
+++ b/modules/term/eshell/config.el
@@ -101,6 +101,10 @@ You should use `set-eshell-alias!' to change this.")
 
   (add-hook 'eshell-mode-hook #'hide-mode-line-mode)
 
+  ;; Remove hscroll-margin in shells, otherwise you get jumpiness when the
+  ;; cursor comes close to the left/right edges of the window.
+  (setq-hook! 'eshell-mode-hook hscroll-margin 0)
+
   ;; Don't auto-write our aliases! Let us manage our own `eshell-aliases-file'
   ;; or configure `+eshell-aliases' via elisp.
   (advice-add #'eshell-write-aliases-list :override #'ignore)
diff --git a/modules/term/term/config.el b/modules/term/term/config.el
index 8429aa392..d3738ebeb 100644
--- a/modules/term/term/config.el
+++ b/modules/term/term/config.el
@@ -9,3 +9,7 @@
 (setq multi-term-dedicated-window-height 20
       multi-term-switch-after-close 'PREVIOUS
       multi-term-buffer-name "doom:terminal")
+
+;; Remove hscroll-margin in shells, otherwise you get jumpiness when the cursor
+;; comes close to the left/right edges of the window.
+(setq-hook! 'term-mode-hook hscroll-margin 0)