[mail] Fix how notmuch and mbsync work

bin/.local/bin/checkmail

@@ -7,9 +7,10 @@ if [ $(pidof $AFEW_SERVICE) ] || [ $(pidof $MBSYNC_SERVICE) ] || [ $(pidof $NOTM
     echo "Checkmail is already running."
 else
     echo "Checking for new mail"
-    afew -m -n
+    mkdir -p ~/Mail/colin@unbl.ink/Junk/{cur,new,tmp}
-    mbsync -a
+    mbsync unblink
     notmuch new
     afew -t -n
+    afew -m -n
 fi
 exit 0

bin/.local/bin/notify-on-new-mail.py

@@ -1,19 +1,91 @@
+#!/usr/bin/env python3
+import http.client
 import json
 import subprocess
+import sys
+import time
 
-new_mail_list = json.loads(
+NTFY_HOST = "ntfy.unbl.ink"
+NTFY_PATH = "/207-comms"
+MAILBOX_URL = "https://box.unbl.ink/mail/?_task=mail&_mbox=INBOX"
+MAX_BODY = 3000
+SEND_DELAY = 1.5
+
+
+def extract_text(body_parts):
+    plain = []
+    html = []
+
+    def walk(parts):
+        for part in parts:
+            content_type = part.get('content-type', '')
+            content = part.get('content')
+            if isinstance(content, list):
+                walk(content)
+            elif content:
+                if 'text/plain' in content_type:
+                    plain.append(content)
+                elif 'text/html' in content_type:
+                    html.append(content)
+
+    walk(body_parts)
+    return '\n'.join(plain) if plain else '\n'.join(html)
+
+
+def send_ntfy(headers, payload):
+    data = payload.encode('utf-8')
+    for attempt in range(3):
+        conn = http.client.HTTPSConnection(NTFY_HOST, timeout=15)
+        try:
+            conn.putrequest("POST", NTFY_PATH, skip_host=True)
+            conn.putheader(b"Host", NTFY_HOST.encode('utf-8'))
+            for name, value in headers.items():
+                conn.putheader(name.encode('utf-8'), value.encode('utf-8'))
+            conn.putheader("Content-Type", b"text/plain; charset=utf-8")
+            conn.putheader("Content-Length", str(len(data)))
+            conn.endheaders(data)
+            resp = conn.getresponse()
+            resp.read()
+            if resp.status == 429:
+                retry = resp.getheader('Retry-After', '5') or '5'
+                time.sleep(float(retry))
+                continue
+            if resp.status < 200 or resp.status >= 300:
+                print(f"ntfy failed: {resp.status}", file=sys.stderr)
+            return
+        except OSError as e:
+            print(f"ntfy error: {e}", file=sys.stderr)
+            time.sleep(attempt + 1)
+        finally:
+            conn.close()
+
+
+def main():
+    new_mail = json.loads(
         subprocess.check_output(
-        ["notmuch", "show", "--format=json", "tag:inbox"],
+            ["notmuch", "show", "--format=json", "tag:new"],
         ).decode('utf-8')
-)
+    )
+
+    for thread in new_mail:
+        for message_pair in thread:
+            headers = message_pair[0]['headers']
+            subject = headers.get('Subject', '')
+            sender = headers.get('From', '')
+            date = headers.get('Date', '')
+            body = extract_text(message_pair[0].get('body', []))[:MAX_BODY]
+
+            payload = f"{sender}\n\n{body}\n\n{date}"
+            send_ntfy(
+                {
+                    "Title": subject,
+                    "Tags": "envelope",
+                    "Click": MAILBOX_URL,
+                },
+                payload,
+            )
+            time.sleep(SEND_DELAY)
 
 
-if new_mail_list:
+if __name__ == "__main__":
-    output = "Mail's here!\n\n"
+    main()
-    for message in new_mail_list:
-        msg = message[0][0]
-        dt = msg['headers']['Date']
-        subj = msg['headers']['Subject']
-        output += f"{dt}\n{subj}\n\n"
-
-    subprocess.run(["dunstify", output])

bin/.local/bin/setup.sh

@@ -170,6 +170,9 @@ UTILS_LINUX=(gnuplot graphviz shfmt shellcheck pandoc)
 
 DESKTOP_LINUX=(kitty alacritty dino syncthing)
 
+# Mail stack (notmuch search, isync sync, afew tagging, msmtp send)
+MAIL_LINUX=(notmuch isync afew msmtp)
+
 FEDORA_EXTRAS=(
 	gnome-extensions-app
 	gnome-tweaks
@@ -210,6 +213,9 @@ MAC_BREW_PKGS=(
 	syncthing
 	pinentry-mac
 	emacs
+	notmuch
+	isync
+	msmtp
 )
 
 MAC_CASKS=(
@@ -246,6 +252,7 @@ fedora)
 	dnf_install_best_effort "${DEV_TOOLS_LINUX[@]}"
 	dnf_install_best_effort "${UTILS_LINUX[@]}"
 	dnf_install_best_effort "${DESKTOP_LINUX[@]}"
+	dnf_install_best_effort "${MAIL_LINUX[@]}"
 	dnf_install_best_effort "${FEDORA_EXTRAS[@]}"
 
 	ensure_flathub
@@ -268,6 +275,7 @@ arch)
 	pacman_install_best_effort "${DEV_TOOLS_LINUX[@]}"
 	pacman_install_best_effort "${UTILS_LINUX[@]}"
 	pacman_install_best_effort "${DESKTOP_LINUX[@]}"
+	pacman_install_best_effort "${MAIL_LINUX[@]}"
 	pacman_install_best_effort "${ARCH_EXTRAS[@]}"
 
 	aur_install_if_possible google-chrome slack-desktop || true
@@ -288,6 +296,11 @@ macos)
 	brew_install "${MAC_BREW_PKGS[@]}"
 	brew_cask "${MAC_CASKS[@]}" || true
 
+	if ! have afew && have pipx; then
+		log "Installing afew via pipx..."
+		pipx install afew || true
+	fi
+
 	if [[ -f "$(brew --prefix)/opt/fzf/install" ]]; then
 		log "Tip: run fzf install helper if you want keybindings/completion:"
 		echo "  $(brew --prefix)/opt/fzf/install"
@@ -319,6 +332,7 @@ freebsd)
 
 	# Optional stuff from your history where it makes sense
 	pkg_install_best_effort "${FREEBSD_MISC[@]}"
+	pkg_install_best_effort "notmuch isync msmtp"
 
 	log "Final system upgrade..."
 	sudo pkg upgrade -y

emacs/.config/doom/config.el

@@ -644,3 +644,10 @@ Always open the result in `eww`."
                       (insert (or body ""))
                       (insert "\n\n")))
                   (pop-to-buffer buf))))))))))
+(setq send-mail-function 'sendmail-send-it
+      sendmail-program "/usr/bin/msmtp"
+      mail-specify-envelope-from t
+      message-sendmail-f-is-evil t
+      message-sendmail-envelope-from 'header
+      message-sendmail-extra-arguments '("--read-envelope-from")
+      mail-envelope-from 'header)

emacs/.config/doom/config.org

@@ -346,7 +346,7 @@ We'll use `msmtp` to send our email. Again, make sure this is installed.
 #+BEGIN_SRC emacs-lisp
 ;; sendmail-program "/usr/local/bin/msmtpq" <--- this doesn't work as advertised right now
 (setq send-mail-function 'sendmail-send-it
-      sendmail-program "/usr/local/bin/msmtp"
+      sendmail-program "/usr/bin/msmtp"
       mail-specify-envelope-from t
       message-sendmail-f-is-evil t
       message-sendmail-envelope-from 'header

mail/.config/afew/config

@@ -1,8 +1,13 @@
 [global]
 
 [SpamFilter]
+[SpammyFilter]
+min_score = 3.0
+max_score = 5.0
+
 [KillThreadsFilter]
-[DMARCReportInspectionFilter]
+# DMARC filter crashes on messages lacking a Subject (afew 4.0.2 + notmuch2)
+#[DMARCReportInspectionFilter]
 
 [FolderNameFilter]
 folder_lowercases = true
@@ -17,43 +22,37 @@ sent_tag = sent
 
 # add specific tags
 
-# Jira tag as Jira
-[Filter.1]
-query = 'from:jira@15five-dev.atlassian.net'
-tags = +jira;-inbox
-message = Jira message
-
 # Orgmode
-[Filter.2]
+[Filter.1]
 query = 'emacs-orgmode'
 tags = -new;-inbox;+orgmode
 message = orgmode message
 
 # Delete some spam
-[Filter.3]
+[Filter.2]
 query = '\'Office365 Message Center\' via Developers OR from:govdelivery@subscriptions.defense.gov OR from:notifications@facebookmai.com OR from:MPP@em.home.dell.com'
 tags = -inbox;+spam
 message = spam message
 
 # Tag interview emails
-[Filter.4]
+[Filter.3]
 query = 'subject:Interview'
 tags = -inbox;+interview
 message = personal message
 
 # Emacs Conf
-[Filter.6]
+[Filter.4]
 query = 'to:emacsconf-discuss@gnu.org'
 tags = -inbox;-new;+emacsconf
 message = emacsconf message
 
-[Filter.7]
+[Filter.5]
 query = 'from:notifications@github.com'
 tags = -inbox;+github
 message = Github message
 
 [MailMover]
-folders = colin@unbl.ink/INBOX colin@unbl.ink/Archive colin@unbl.ink/Trash colin@castine.town/INBOX colin@castine.town/Archive colin@castine.town/Trash
+folders = colin@unbl.ink/INBOX colin@unbl.ink/Archive colin@unbl.ink/Trash
 
 rename = True
 max_age = 15
@@ -61,11 +60,3 @@ max_age = 15
 colin@unbl.ink/INBOX = 'tag:spam':colin@unbl.ink/Junk 'NOT tag:inbox':colin@unbl.ink/Archive 'tag:trash':colin@unbl.ink/Trash
 colin@unbl.ink/Archive = 'tag:inbox':colin@unbl.ink/INBOX 'tag:trash':colin@unbl.ink/Trash
 colin@unbl.ink/Trash = 'tag:inbox':colin@unbl.ink/INBOX 'tag:archive':colin@unbl.ink/Archive
-
-colin@castine.town/INBOX = 'tag:spam':colin@castine.town/Junk 'NOT tag:inbox':colin@castine.town/Archive 'tag:trash':colin@castine.town/Trash
-colin@castine.town/Archive = 'tag:inbox':colin@castine.town/INBOX 'tag:trash':colin@castine.town/Trash
-colin@castine.town/Trash = 'tag:inbox':colin@castine.town/INBOX 'tag:archive':colin@castine.town/Archive
-
-secstate@sdf.org/INBOX = 'tag:spam':secstate@sdf.org/Junk 'NOT tag:inbox':secstate@sdf.org/Archive 'tag:trash':secstate@sdf.org/Trash
-secstate@sdf.org/Archive = 'tag:inbox':secstate@sdf.org/INBOX 'tag:trash':secstate@sdf.org/Trash
-secstate@sdf.org/Trash = 'tag:inbox':secstate@sdf.org/INBOX 'tag:archive':secstate@sdf.org/Archive

mail/.config/afew/spammy_filter.py

@@ -0,0 +1,21 @@
+from afew.filters.BaseFilter import Filter
+from afew.FilterRegistry import register_filter
+
+@register_filter
+class SpammyFilter(Filter):
+    message = 'Tag borderline spam (X-Spam-Score in gray zone)'
+    min_score = 3.0
+    max_score = 5.0
+
+    def __init__(self, database, **kwargs):
+        super().__init__(database, **kwargs)
+        self.min_score = float(self.min_score)
+        self.max_score = float(self.max_score)
+
+    def handle_message(self, message):
+        try:
+            score = float(message.header('X-Spam-Score'))
+        except (LookupError, ValueError, TypeError):
+            return
+        if self.min_score <= score < self.max_score:
+            self.add_tags(message, 'spammy')

mail/.config/notmuch/default/hooks/post-new

@@ -1,2 +1,3 @@
 #!/bin/sh
-python ~/.bin/notify-on-new-mail.py
+python ~/.local/bin/notify-on-new-mail.py
+exit 0

mail/.mbsyncrc

@@ -2,7 +2,7 @@ IMAPAccount unblink
 Host box.unbl.ink
 User colin@unbl.ink
 PassCmd "pass personal/colin@unbl.ink"
-SSLType IMAPS
+TLSType IMAPS
 Port 993
 
 IMAPStore unblink-remote
@@ -15,48 +15,22 @@ Inbox ~/Mail/colin@unbl.ink/INBOX
 Trash Trash
 
 Channel unblink-folders
-Master :unblink-remote:
+Far :unblink-remote:
-Slave :unblink-local:
+Near :unblink-local:
 Patterns "INBOX" "Drafts" "Archive" "Sent" "Trash"
 Create Both
 Expunge Both
 SyncState *
+MaxSize 1m
 
 Group unblink 
 Channel unblink-folders
 
-IMAPAccount castine
-Host box.castine.town
-User colin@castine.town
-PassCmd "pass personal/colin@castine.town"
-SSLType IMAPS
-Port 993
-
-IMAPStore castine-remote
-account castine 
-
-MaildirStore castine-local
-# trailing "/" is important
-Path ~/Mail/colin@castine.town/
-Inbox ~/Mail/colin@castine.town/INBOX
-Trash Trash
-
-Channel castine-folders
-Master :castine-remote:
-Slave :castine-local:
-Patterns "INBOX" "Drafts" "Archive" "Sent" "Trash"
-Create Both
-Expunge Both
-SyncState *
-
-Group castine
-Channel castine-folders
-
 IMAPAccount sdf
 Host mx.sdf.org
 User secstate
 PassCmd "pass personal/secstate@sdf.org"
-SSLType IMAPS
+TLSType IMAPS
 Port 993
 
 IMAPStore sdf-remote
@@ -69,53 +43,13 @@ Inbox ~/Mail/secstate@sdf.org/INBOX
 Trash Trash
 
 Channel sdf-folders
-Master :sdf-remote:
+Far :sdf-remote:
-Slave :sdf-local:
+Near :sdf-local:
 Patterns "INBOX" "Drafts" "Sent"
 Create Both
 Expunge Both
 SyncState *
+MaxSize 1m
 
 Group sdf
 Channel sdf-folders
-
-IMAPAccount 15five
-Host imap.gmail.com
-User colin.powell@15five.com
-PassCmd "pass 15five/mbsync"
-SSLType IMAPS
-CertificateFile /usr/local/share/certs/ca-root-nss.crt
-Port 993
-
-IMAPStore 15five-remote
-Account 15five
-
-MaildirStore 15five-local
-SubFolders Legacy
-Path ~/Mail/colin.powell@15five.com/
-Inbox ~/Mail/colin.powell@15five.com/Inbox
-
-Channel 15five-default
-Master :15five-remote:
-Slave :15five-local:
-Patterns * ![Gmail]* 
-Create Both
-SyncState *
-Sync All
-
-Channel 15five-sent
-Master :15five-remote:"[Gmail]/Sent Mail"
-Slave :15five-local:Sent
-Create Slave
-Sync Pull
-
-Channel 15five-trash
-Master :15five-remote:"[Gmail]/Trash"
-Slave :15five-local:Trash
-Create Slave
-Sync Pull
-
-Group 15five
-Channel 15five-default
-Channel 15five-sent
-Channel 15five-trash

mail/.msmtprc

@@ -17,26 +17,6 @@ passwordeval   pass personal/colin@unbl.ink
 tls_fingerprint 63:EB:0D:1E:6D:23:2F:24:C8:58:86:28:6B:99:2D:F7:B9:91:97:BC:B4:56:47:C8:47:2C:19:10:0C:E7:DC:7D
 #############################
 
-#############################
-account        castine.town
-auth           on
-host           box.castine.town
-port           587
-from           colin@castine.town
-user           colin@castine.town
-passwordeval   pass personal/colin@castine.town
-tls_fingerprint EF:3E:CD:72:F5:BD:63:37:F3:19:F6:1A:5E:B7:BC:32:D7:0C:DB:D8:37:64:62:3F:44:FB:2E:72:1F:69:B2:D9
-#############################
-
-#############################
-account        15five
-auth           on
-host           smtp.gmail.com
-port           587
-from           colin.powell@15five.com
-user           colin.powell@15five.com
-passwordeval   pass 15five/mbsync
-#############################
 #
 account default : unblink
 

mail/.notmuch-config

@@ -4,17 +4,20 @@ path=Mail
 [user]
 name=Colin Powell
 primary_email=colin@unbl.ink
-other_email=colin@unbl.ink;colin@castine.town;colin@unbl.ink
+other_email=colin.powell@gmail.com;colin@castine.town;secstate@sdf.org
 
 [new]
 tags=new
 ignore=.mbsyncstate;.isncuidmap.db;.uidvalidity
 
+[index]
+header.List-Id=list
+
 [search]
-exclude_tags=spam;muted
+exclude_tags=spam;muted;deleted;trash
 
 [maildir]
 synchronize_flags=true
 
 [crypto]
-gpg_path=/usr/local/bin/gpg
+gpg_path=/usr/bin/gpg

systemd/.config/systemd/user/syncmail.service

@@ -2,7 +2,7 @@
 Description=Sync mail
 
 [Service]
-ExecStart=/home/powellc/.bin/syncmail
+ExecStart=/home/powellc/.local/bin/checkmail
 
 [Install]
 WantedBy=default.target