emenel/doomemacs
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Rewrite core-cli

Browse source 
Highlights:
- 'doom purge' now purges builds, elpa packages, and repos by default.
  Regrafting repos is now opt-in with the -g/--regraft switches.
  Negation flags have been added for elpa/repos: -e/--no-elpa and
  -r/--no-repos.
- Removed 'doom rebuild' (it is now just 'doom build' or 'doom b').
- Removed 'doom build's -f flag, this is now the default. Added the -r
  flag instead, which only builds packages that need rebuilding.
- 'doom update' now updates packages synchronously, but produces more
  informative output about the updating process.
- Straight can now prompt in batch mode, which resolves a lot of issues
  with 'doom update' (and 'doom upgrade') freezing indefinitely or
  throwing repo branch errors.
- 'bin/doom's switches are now positional. Switches aimed at `bin/doom`
  must precede any subcommands. e.g.
    Do: 'doom -yd upgrade'
    Don't do: 'doom upgrade -yd'
- Moved 'doom doctor' from bin/doom-doctor to core/cli/doctor, and
  integrated core/doctor.el into it, as to avoid naming conflicts
  between it and Emacs doctor.
- The defcli! macro now has a special syntax for declaring flags, their
  arguments and descriptions.

Addresses #1981, #1925, #1816, #1721, #1322
This commit is contained in:
Henrik Lissner 2019-11-07 15:59:56 -05:00
parent 99cd52e70f
commit 873fc5c0db
No known key found for this signature in database
GPG key ID: 5F6C0EA160557395
16 changed files with 996 additions and 1266 deletions
Download patch file Download diff file Expand all files Collapse all files

464
core/cli/packages.el
View file

@ -1,73 +1,55 @@
;; -*- no-byte-compile: t; -*-
;;; core/cli/packages.el
(defmacro doom--ensure-autoloads-while (&rest body)
`(progn
(straight-check-all)
(doom-reload-core-autoloads)
(when (progn ,@body)
(doom-reload-package-autoloads 'force-p))
t))
;;
;;; Dispatchers
(defcli! (update u) (&rest args)
(defcli! (update u) ()
"Updates packages.
This works by fetching all installed package repos and checking the distance
between HEAD and FETCH_HEAD. This can take a while.
This excludes packages whose `package!' declaration contains a non-nil :freeze
or :ignore property.
or :ignore property."
(straight-check-all)
(doom-cli-reload-core-autoloads)
(when (doom-cli-packages-update)
(doom-cli-reload-package-autoloads 'force-p))
t)
Switches:
-t/--timeout TTL Seconds until a thread is timed out (default: 45)
--threads N How many threads to use (default: 8)"
(doom--ensure-autoloads-while
(doom-packages-update
doom-auto-accept
(when-let (threads (cadr (member "--threads" args)))
(string-to-number threads))
(when-let (timeout (cadr (or (member "--timeout" args)
(member "-t" args))))
(string-to-number timeout)))))
(defcli! (rebuild build b) (&rest args)
"Rebuilds all installed packages.
(defcli! (build b)
((rebuild-p ["-r"] "Only rebuild packages that need rebuilding"))
"Byte-compiles & symlinks installed packages.
This ensures that all needed files are symlinked from their package repo and
their elisp files are byte-compiled.
their elisp files are byte-compiled. This is especially necessary if you upgrade
Emacs (as byte-code is generally not forward-compatible)."
(when (doom-cli-packages-build (not rebuild-p))
(doom-cli-reload-package-autoloads 'force-p))
t)
Switches:
-f Forcibly rebuild autoloads files, even if they're up-to-date"
(doom--ensure-autoloads-while
(doom-packages-rebuild doom-auto-accept (member "-f" args))))
(defcli! (purge p)
((nobuilds-p ["-b" "--no-builds"] "Don't purge unneeded (built) packages")
(noelpa-p ["-p" "--no-elpa"] "Don't purge ELPA packages")
(norepos-p ["-r" "--no-repos"] "Don't purge unused straight repos")
(regraft-p ["-g" "--regraft"] "Regraft git repos (ie. compact them)"))
"Deletes orphaned packages & repos, and compacts them.
(defcli! (purge p) (&rest args)
"Deletes any unused ELPA packages, straight builds, and (optionally) repos.
Purges all installed ELPA packages (as they are considered temporary). Purges
all orphaned package repos and builds. If -g/--regraft is supplied, the git
repos among them will be regrafted and compacted to ensure they are as small as
possible.
By default, this does not purge ELPA packages or repos. It is a good idea to run
'doom purge --all' once in a while, to stymy build-up of repos and ELPA
packages that could be taking up precious space.
It is a good idea to occasionally run this doom purge -g to ensure your package
list remains lean."
(straight-check-all)
(when (doom-cli-packages-purge
(not noelpa-p)
(not norepos-p)
(not nobuilds-p)
regraft-p)
(doom-cli-reload-package-autoloads 'force-p))
t)
Switches:
--no-builds Don't purge unneeded (built) packages
-e / --elpa Don't purge ELPA packages
-r / --repos Purge unused repos
--all Purge builds, elpa packages and repos"
(doom--ensure-autoloads-while
(doom-packages-purge (or (member "-e" args)
(member "--elpa" args)
(member "--all" args))
(not (member "--no-builds" args))
(or (member "-r" args)
(member "--repos" args)
(member "--all" args))
doom-auto-accept)))
;; (defcli! rollback () ; TODO rollback
;; (defcli! rollback () ; TODO doom rollback
;; "<Not implemented yet>"
;; (user-error "Not implemented yet, sorry!"))
@ -75,15 +57,12 @@ Switches:
;;
;;; Library
(defun doom-packages-install (&optional auto-accept-p)
(defun doom-cli-packages-install ()
"Installs missing packages.
This function will install any primary package (i.e. a package with a `package!'
declaration) or dependency thereof that hasn't already been.
Unless AUTO-ACCEPT-P is non-nil, this function will prompt for confirmation with
a list of packages that will be installed."
(print! "> Installing & building packages...")
declaration) or dependency thereof that hasn't already been."
(print! (start "Installing & building packages..."))
(print-group!
(let ((n 0))
(dolist (package (hash-table-keys straight--recipe-cache))
@ -91,7 +70,7 @@ a list of packages that will be installed."
(local-repo)
(let ((existed-p (file-directory-p (straight--repos-dir package))))
(condition-case-unless-debug e
(and (straight-use-package (intern package) nil nil " ")
(and (straight-use-package (intern package) nil nil (make-string (1- (or doom-format-indent 1)) 32))
(not existed-p)
(file-directory-p (straight--repos-dir package))
(cl-incf n))
@ -104,17 +83,18 @@ a list of packages that will be installed."
t))))
(defun doom-packages-rebuild (&optional auto-accept-p all)
(defun doom-cli-packages-build (&optional force-p)
"(Re)build all packages."
(print! (start "(Re)building %spackages...") (if all "all " ""))
(print! (start "(Re)building %spackages...") (if force-p "all " ""))
(print-group!
(let ((n 0))
(if all
(if force-p
(let ((straight--packages-to-rebuild :all)
(straight--packages-not-to-rebuild (make-hash-table :test #'equal)))
(dolist (package (hash-table-keys straight--recipe-cache))
(straight-use-package
(intern package) nil (lambda (_) (cl-incf n) nil) " ")))
(intern package) nil (lambda (_) (cl-incf n) nil)
(make-string (1- (or doom-format-indent 1)) 32))))
(dolist (recipe (hash-table-values straight--recipe-cache))
(straight--with-plist recipe (package local-repo no-build)
(unless (or no-build (null local-repo))
@ -139,7 +119,9 @@ a list of packages that will be installed."
(lambda (&rest _) (cl-incf n)))
(let ((straight--packages-to-rebuild :all)
(straight--packages-not-to-rebuild (make-hash-table :test #'equal)))
(straight-use-package (intern package) nil nil " "))
(straight-use-package
(intern package) nil nil
(make-string (or doom-format-indent 0) 32)))
(straight--byte-compile-package recipe)
(dolist (dep (straight--get-dependencies package))
(when-let (recipe (gethash dep straight--recipe-cache))
@ -151,268 +133,107 @@ a list of packages that will be installed."
t))))
(defun doom--packages-remove-outdated-f (packages)
(async-start
`(lambda ()
(setq load-path ',load-path
doom-modules ',doom-modules
user-emacs-directory ',user-emacs-directory)
(condition-case e
(let (packages errors)
(load ,(concat doom-core-dir "core.el"))
(doom-initialize 'force)
(dolist (recipe ',group)
(when (straight--repository-is-available-p recipe)
(straight-vc-git--destructure recipe
(package local-repo nonrecursive upstream-remote upstream-repo upstream-host branch)
(condition-case e
(let ((default-directory (straight--repos-dir local-repo)))
;; HACK We normalize packages to avoid certain scenarios
;; where `straight-fetch-package' will create an
;; interactive popup prompting for action (which will
;; cause this async process to block indefinitely). We
;; can't use `straight-normalize-package' because could
;; create popup prompts too, so we do it manually:
(shell-command-to-string "git merge --abort")
(straight--get-call "git" "reset" "--hard" branch)
(straight--get-call "git" "clean" "-ffd")
(unless nonrecursive
(shell-command-to-string "git submodule update --init --recursive"))
(when upstream-repo
(let ((desired-url (straight-vc-git--encode-url upstream-repo upstream-host))
(actual-url (condition-case nil
(straight--get-call "git" "remote" "get-url" upstream-remote)
(error nil))))
(unless (straight-vc-git--urls-compatible-p actual-url desired-url)
(straight--get-call "git" "remote" "remove" upstream-remote)
(straight--get-call "git" "remote" "add" upstream-remote desired-url)
(straight--get-call "git" "fetch" upstream-remote))))
(straight-fetch-package package)
;; REVIEW Is there no better way to get this information?
(let ((n (length
(split-string
(straight--get-call "git" "rev-list" "--left-right" "HEAD..@{u}")
"\n" t)))
(pretime
(string-to-number
(shell-command-to-string "git log -1 --format=%at HEAD")))
(time
(string-to-number
;; HACK `straight--get-call' has a higher failure
;; rate when querying FETCH_HEAD; not sure why.
;; Doing this manually, with
;; `shell-command-to-string' works fine.
(shell-command-to-string "git log -1 --format=%at FETCH_HEAD"))))
(with-current-buffer (straight--process-get-buffer)
(with-silent-modifications
(print! (debug (autofill "%s") (indent 2 (buffer-string))))
(erase-buffer)))
(when (> n 0)
(push (list n pretime time recipe)
packages))))
(error
(push (list package e (string-trim (or (straight--process-get-output) "")))
errors))))))
(if errors
(cons 'error errors)
(cons 'ok (nreverse packages))))
(error
(cons 'error e))))))
(defun doom-packages-update (&optional auto-accept-p threads timeout)
"Updates packages.
Unless AUTO-ACCEPT-P is non-nil, this function will prompt for confirmation with
a list of packages that will be updated."
(print! (start "Scanning for outdated packages (this may take a while)..."))
(print-group!
(when timeout
(print! (info "Using %S as timeout value" timeout)))
(when threads
(print! (info "Limiting to %d thread(s)" threads)))
;; REVIEW Does this fail gracefully enough? Is it error tolerant?
;; TODO Add version-lock checks; don't want to spend all this effort on
;; packages that shouldn't be updated
(let* ((futures
;; REVIEW We can do better "thread" management here
(or (cl-loop for group
in (seq-partition (hash-table-values straight--repo-cache)
(/ (hash-table-count straight--repo-cache)
(or threads 8)))
for future = (doom--packages-remove-outdated-f group)
if (processp future)
collect (cons future group)
else
do (print! (warn "Failed to create thread for:\n\n%s\n\nReason: %s"
group future)))
(error! "Failed to create any threads")))
(total (length futures))
(timeout (or timeout 45)))
(condition-case-unless-debug e
(let (specs)
(while futures
(print! ". %.0f%%" (* (/ (- total (length futures))
(float total))
100))
(let ((time 0))
(catch 'timeout
(while (not (async-ready (caar futures)))
(when (> time timeout)
(print! (warn "A thread has timed out. The following packages were skipped: %s"
(mapconcat (lambda (p) (plist-get p :package))
(cdar futures)
", ")))
(throw 'timeout (pop futures)))
(sleep-for 1)
(when (cl-evenp time)
(print! "."))
(cl-incf time))
(cl-destructuring-bind (status . result)
(or (async-get (car (pop futures)))
(cons nil nil))
(cond ((null status)
(error "Thread returned an invalid result: %S" errors))
((eq status 'error)
(error "There were errors:\n\n%s"
(cond ((and (listp result)
(symbolp (car result)))
(prin1-to-string result))
((stringp result)
result)
((mapconcat (lambda (e)
(format! " - %s: %s" (yellow (car e)) (cdr e)))
result
"\n")))))
((eq status 'ok)
(print! (debug "Appended %S to package list") (or result "nothing"))
(appendq! specs result))
((error "Thread returned a non-standard status: %s\n\n%s"
status result)))))))
(print! ". 100%%")
(terpri)
(if-let (specs (delq nil specs))
(if (not
(or auto-accept-p
(y-or-n-p
(format!
"%s\n\nThere %s %d package%s available to update. Update them?"
(mapconcat
(lambda (spec)
(cl-destructuring-bind (n pretime time recipe) spec
(straight--with-plist recipe (package)
(format! "+ %-33s %s commit(s) behind %s -> %s"
(yellow package) (yellow n)
(format-time-string "%Y%m%d" pretime)
(format-time-string "%Y%m%d" time)))))
specs
"\n")
(if (cdr specs) "are" "is")
(length specs)
(if (cdr specs) "s" "")))))
(ignore (print! (info "Aborted update")))
(terpri)
(straight--make-package-modifications-available)
(let ((straight--packages-to-rebuild (make-hash-table :test #'equal))
(straight--packages-not-to-rebuild (make-hash-table :test #'equal)))
(dolist (spec specs)
(cl-destructuring-bind (n pretime time recipe) spec
(straight--with-plist recipe (local-repo package)
(let ((default-directory (straight--repos-dir local-repo)))
(print! (start "Updating %S") package)
(straight-merge-package package)
;; HACK `straight-rebuild-package' doesn't pick up that
;; this package has changed, so we do it manually. Is
;; there a better way?
(ignore-errors
(delete-directory (straight--build-dir package) 'recursive))
(puthash package t straight--packages-to-rebuild)
(cl-incf n))
(with-current-buffer (straight--process-get-buffer)
(with-silent-modifications
(print! (debug (autofill "%s") (indent 2 (buffer-string))))
(erase-buffer))))))
(doom--finalize-straight)
(doom-packages-rebuild auto-accept-p))
t)
(print! (success "No packages to update"))
nil))
(error
(message "Output:\n%s" (straight--process-get-output))
(signal (car e) (error-message-string e)))))))
(defun doom-cli-packages-update ()
"Updates packages."
(print! (start "Updating packages (this may take a while)..."))
(let ((straight--packages-to-rebuild (make-hash-table :test #'equal))
(total (hash-table-count straight--repo-cache))
(i 1)
errors)
(print-group!
(dolist (recipe (hash-table-values straight--repo-cache))
(straight--with-plist recipe (package type local-repo)
(condition-case-unless-debug e
(let* ((default-directory (straight--repos-dir local-repo))
(commit (straight-vc-get-commit type local-repo)))
(if (not (straight-vc-fetch-from-remote recipe))
(print! (warn "(%d/%d) Failed to fetch %s" i total package))
(let ((output (straight--process-get-output)))
(straight-merge-package package)
(let ((newcommit (straight-vc-get-commit type local-repo)))
(if (string= commit newcommit)
(print! (info "(%d/%d) %s is up-to-date") i total package)
(ignore-errors
(delete-directory (straight--build-dir package) 'recursive))
(puthash package t straight--packages-to-rebuild)
(print! (success "(%d/%d) %s updated (%s -> %s)") i total package
(substring commit 0 7)
(substring newcommit 0 7))
(unless (string-empty-p output)
(print-group!
(print! (info "%s") output)
(when (eq type 'git)
(straight--call "git" "log" "--oneline" newcommit (concat "^" commit))
(print-group!
(print! "%s" (straight--process-get-output))))))))))
(cl-incf i))
(user-error
(signal 'user-error (error-message-string e)))
(error
(print! (warn "(%d/%d) Encountered error with %s" i total package))
(print-group!
(print! (error "%s" e))
(print-group! (print! (info "%s" (straight--process-get-output)))))
(push package errors)))))
(when errors
(print! (error "There were %d errors, the offending packages are: %s")
(length errors) (string-join errors ", ")))
(if (hash-table-empty-p straight--packages-to-rebuild)
(ignore
(print! (success "All %d packages are up-to-date")
(hash-table-count straight--repo-cache)))
(let ((count (hash-table-count straight--packages-to-rebuild))
(packages (hash-table-keys straight--packages-to-rebuild)))
(sort packages #'string-lessp)
(doom--finalize-straight)
(doom-cli-packages-build)
(print! (success "Updated %d package(s)") count))
t))))
;;; PURGE (for the emperor)
(defun doom--prompt-p (list-fn list preamble postamble)
(or (y-or-n-p (format "%s%s\n\n%s"
(if preamble (concat preamble "\n\n") "")
(mapconcat list-fn list "\n")
(or postamble "")))
(user-error! "Aborted")))
(defun doom--prompt-columns-p (row-fn list preamble postamble)
(doom--prompt-p (lambda (row)
(mapconcat row-fn row ""))
(seq-partition (cl-sort (copy-sequence list) #'string-lessp)
3)
preamble
postamble))
(defun doom--packages-purge-build (build)
(defun doom--cli-packages-purge-build (build)
(let ((build-dir (straight--build-dir build)))
(print! (start "Purging build/%s..." build))
(delete-directory build-dir 'recursive)
(if (file-directory-p build-dir)
(ignore (print! (error "Failed to purg build/%s" build)))
(print! (success "Purged build/%s" build))
t)))
(defun doom--packages-purge-builds (builds &optional auto-accept-p)
(defun doom--cli-packages-purge-builds (builds)
(if (not builds)
(progn (print! (info "No builds to purge"))
0)
(or auto-accept-p
(doom--prompt-columns-p
(lambda (p) (format " + %-20.20s" p)) builds nil
(format! "Found %d orphaned package builds. Purge them?"
(length builds))))
(length
(delq nil (mapcar #'doom--packages-purge-build builds)))))
(delq nil (mapcar #'doom--cli-packages-purge-build builds)))))
(defun doom--packages-regraft-repo (repo)
(defun doom--cli-packages-regraft-repo (repo)
(let ((default-directory (straight--repos-dir repo)))
(if (not (file-directory-p ".git"))
(ignore (print! (warn "repos/%s is not a git repo, skipping" repo)))
(print! (debug "Regrafting repos/%s..." repo))
(straight--call "git" "reset" "--hard")
(straight--call "git" "clean" "--ffd")
(straight--call "git" "replace" "--graft" "HEAD")
(straight--call "git" "gc")
(print! (debug "%s" (straight--process-get-output)))
(print! (success "Regrafted repos/%s" repo))
(let ((before-size (doom-directory-size default-directory)))
(straight--call "git" "reset" "--hard")
(straight--call "git" "clean" "-ffd")
(if (not (car (straight--call "git" "replace" "--graft" "HEAD")))
(print! (info "repos/%s is already compact" repo))
(straight--call "git" "gc")
(print! (success "Regrafted repos/%s (from %0.1fKB to %0.1fKB)")
repo before-size (doom-directory-size default-directory))
(print-group! (print! "%s" (straight--process-get-output)))))
t)))
(defun doom--packages-regraft-repos (repos &optional auto-accept-p)
(defun doom--cli-packages-regraft-repos (repos)
(if (not repos)
(progn (print! (info "No repos to regraft"))
0)
(or auto-accept-p
(y-or-n-p (format! "Preparing to regraft all %d repos. Continue?"
(length repos)))
(user-error! "Aborted!"))
(if (executable-find "du")
(cl-destructuring-bind (status . size)
(doom-sh "du" "-sh" (straight--repos-dir))
(prog1 (delq nil (mapcar #'doom--packages-regraft-repo repos))
(cl-destructuring-bind (status . newsize)
(doom-sh "du" "-sh" (straight--repos-dir))
(print! (success "Finshed regrafted. Size before: %s and after: %s"
(car (split-string size "\t"))
(car (split-string newsize "\t")))))))
(delq nil (mapcar #'doom--packages-regraft-repo repos)))))
(let ((before-size (doom-directory-size (straight--repos-dir))))
(prog1 (print-group! (delq nil (mapcar #'doom--cli-packages-regraft-repo repos)))
(let ((after-size (doom-directory-size (straight--repos-dir))))
(print! (success "Finished regrafting. Size before: %0.1fKB and after: %0.1fKB (-%0.1fKB)")
before-size after-size
(- after-size before-size)))))))
(defun doom--packages-purge-repo (repo)
(print! (debug "Purging repos/%s..." repo))
(defun doom--cli-packages-purge-repo (repo)
(let ((repo-dir (straight--repos-dir repo)))
(delete-directory repo-dir 'recursive)
(ignore-errors
@ -422,19 +243,14 @@ a list of packages that will be updated."
(print! (success "Purged repos/%s" repo))
t)))
(defun doom--packages-purge-repos (repos &optional auto-accept-p)
(defun doom--cli-packages-purge-repos (repos)
(if (not repos)
(progn (print! (info "No repos to purge"))
0)
(or auto-accept-p
(doom--prompt-columns-p
(lambda (p) (format " + %-20.20s" p)) repos nil
(format! "Found %d orphaned repos. Purge them?"
(length repos))))
(length
(delq nil (mapcar #'doom--packages-purge-repo repos)))))
(delq nil (mapcar #'doom--cli-packages-purge-repo repos)))))
(defun doom--packages-purge-elpa (&optional auto-accept-p)
(defun doom--cli-packages-purge-elpa ()
(unless (bound-and-true-p package--initialized)
(package-initialize))
(let ((packages (cl-loop for (package desc) in package-alist
@ -444,16 +260,11 @@ a list of packages that will be updated."
(if (not package-alist)
(progn (print! (info "No ELPA packages to purge"))
0)
(doom--prompt-columns-p
(lambda (p) (format " + %-20.20s" p))
(mapcar #'car packages) nil
(format! "Found %d orphaned ELPA packages. Purge them?"
(length package-alist)))
(mapc (doom-rpartial #'delete-directory 'recursive)
(mapcar #'cdr packages))
(length packages))))
(defun doom-packages-purge (&optional elpa-p builds-p repos-p auto-accept-p)
(defun doom-cli-packages-purge (&optional elpa-p builds-p repos-p regraft-repos-p)
"Auto-removes orphaned packages and repos.
An orphaned package is a package that isn't a primary package (i.e. doesn't have
@ -461,10 +272,7 @@ a `package!' declaration) or isn't depended on by another primary package.
If BUILDS-P, include straight package builds.
If REPOS-P, include straight repos.
If ELPA-P, include packages installed with package.el (M-x package-install).
Unless AUTO-ACCEPT-P is non-nil, this function will prompt for confirmation with
a list of packages that will be removed."
If ELPA-P, include packages installed with package.el (M-x package-install)."
(print! (start "Searching for orphaned packages to purge (for the emperor)..."))
(cl-destructuring-bind (&optional builds-to-purge repos-to-purge repos-to-regraft)
(let ((rdirs (straight--directory-files (straight--repos-dir) nil nil 'sort))
@ -479,18 +287,20 @@ a list of packages that will be removed."
(print-group!
(if (not builds-p)
(print! (info "Skipping builds"))
(and (/= 0 (doom--packages-purge-builds builds-to-purge auto-accept-p))
(and (/= 0 (doom--cli-packages-purge-builds builds-to-purge))
(setq success t)
(straight-prune-build-cache)))
(if (not elpa-p)
(print! (info "Skipping elpa packages"))
(and (/= 0 (doom--packages-purge-elpa auto-accept-p))
(and (/= 0 (doom--cli-packages-purge-elpa))
(setq success t)))
(if (not repos-p)
(print! (info "Skipping repos"))
(and (/= 0 (doom--packages-purge-repos repos-to-purge auto-accept-p))
(setq success t))
(and (doom--packages-regraft-repos repos-to-regraft auto-accept-p)
(and (/= 0 (doom--cli-packages-purge-repos repos-to-purge))
(setq success t)))
(if (not regraft-repos-p)
(print! (info "Skipping regrafting"))
(and (doom--cli-packages-regraft-repos repos-to-regraft)
(setq success t)))
(when success
(doom--finalize-straight)