matrix: working E2EE, on by default

mautrix-go now include a cryptohelper package, which solves all my
issues and just works. the setting is now on by default, however
packages are not yet built with it.

api-messages.go

@@ -614,7 +614,7 @@ func (app *appContext) MatrixConnect(gc *gin.Context) {
 	if app.storage.GetMatrix() == nil {
 		app.storage.deprecatedMatrix = matrixStore{}
 	}
-	roomID, encrypted, err := app.matrix.CreateRoom(req.UserID)
+	roomID, err := app.matrix.CreateRoom(req.UserID)
 	if err != nil {
 		app.err.Printf(lm.FailedCreateRoom, err)
 		respondBool(500, false, gc)
@@ -625,9 +625,7 @@ func (app *appContext) MatrixConnect(gc *gin.Context) {
 		RoomID:  string(roomID),
 		Lang:    "en-us",
 		Contact: true,
-		Encrypted: encrypted,
 	})
-	app.matrix.isEncrypted[roomID] = encrypted
 	respondBool(200, true, gc)
 }
 

auth.go

@@ -43,7 +43,7 @@ func (app *appContext) webAuth() gin.HandlerFunc {
 	return app.authenticate
 }
 
-func (app *appContext) authLog(v any) { app.debug.Printf(lm.FailedAuthRequest, v) }
+func (app *appContext) authLog(v any) { app.debug.PrintfCustomLevel(4, lm.FailedAuthRequest, v) }
 
 // CreateToken returns a web token as well as a refresh token, which can be used to obtain new tokens.
 func CreateToken(userId, jfId string, admin bool) (string, string, error) {
@@ -270,7 +270,7 @@ func (app *appContext) decodeValidateRefreshCookie(gc *gin.Context, cookieName s
 	}
 	token, err := jwt.Parse(cookie, checkToken)
 	if err != nil {
-		app.authLog(lm.FailedParseJWT)
+		app.authLog(fmt.Sprintf(lm.FailedParseJWT, err))
 		respond(400, lm.InvalidJWT, gc)
 		return
 	}

backups.go

@@ -170,5 +170,6 @@ func newBackupDaemon(app *appContext) *GenericDaemon {
 			app.makeBackup()
 		},
 	)
+	d.Name("Backup")
 	return d
 }

config/config-base.json

@@ -1297,14 +1297,14 @@
                     "description": "Default Matrix message language. Visit weblate if you'd like to translate."
                 },
                 "encryption": {
-                    "name": "End-to-end encryption (experimental)",
+                    "name": "End-to-end encryption",
                     "required": false,
                     "requires_restart": true,
                     "depends_true": "enabled",
-                    "advanced": true,
+                    "advanced": false,
                     "type": "bool",
-                    "value": false,
+                    "value": true,
-                    "description": "Enable end-to-end encryption for messages. Very experimental, currently does not support receiving commands (e.g !lang)."
+                    "description": "Enable end-to-end encryption for messages."
                 }
             }
         },

go.mod

@@ -45,6 +45,7 @@ require (
 	github.com/itchyny/timefmt-go v0.1.6
 	github.com/lithammer/shortuuid/v3 v3.0.7
 	github.com/mailgun/mailgun-go/v4 v4.14.0
+	github.com/mattn/go-sqlite3 v1.14.22
 	github.com/robert-nix/ansihtml v1.0.1
 	github.com/steambap/captcha v1.4.1
 	github.com/swaggo/files v1.0.1

housekeeping-d.go

@@ -131,12 +131,12 @@ func newHousekeepingDaemon(interval time.Duration, app *appContext) *GenericDaem
 		func(app *appContext) { app.clearActivities() },
 	)
 
-	d.Name("Housekeeping daemon")
+	d.Name("Housekeeping")
 
 	clearEmail := app.config.Section("email").Key("require_unique").MustBool(false)
-	clearDiscord := app.config.Section("discord").Key("require_unique").MustBool(false) || app.config.Section("discord").Key("disable_enable_role").MustBool(false)
+	clearDiscord := discordEnabled && (app.config.Section("discord").Key("require_unique").MustBool(false) || app.config.Section("discord").Key("disable_enable_role").MustBool(false))
-	clearTelegram := app.config.Section("telegram").Key("require_unique").MustBool(false)
+	clearTelegram := telegramEnabled && (app.config.Section("telegram").Key("require_unique").MustBool(false))
-	clearMatrix := app.config.Section("matrix").Key("require_unique").MustBool(false)
+	clearMatrix := matrixEnabled && (app.config.Section("matrix").Key("require_unique").MustBool(false))
 	clearPWR := app.config.Section("captcha").Key("enabled").MustBool(false) && !app.config.Section("captcha").Key("recaptcha").MustBool(false)
 
 	if clearEmail || clearDiscord || clearTelegram || clearMatrix {

jellyseerr-d.go

@@ -77,6 +77,6 @@ func newJellyseerrDaemon(interval time.Duration, app *appContext) *GenericDaemon
 			app.SynchronizeJellyseerrUsers()
 		},
 	)
-	d.Name("Jellyseerr import daemon")
+	d.Name("Jellyseerr import")
 	return d
 }

logger/logger.go

@@ -84,6 +84,18 @@ func (l *Logger) Printf(format string, v ...interface{}) {
 	l.logger.Print(out)
 }
 
+func (l *Logger) PrintfCustomLevel(level int, format string, v ...interface{}) {
+	if l.empty {
+		return
+	}
+	var out string
+	if l.shortfile {
+		out = Lshortfile(level)
+	}
+	out += " " + l.printer.Sprintf(format, v...)
+	l.logger.Print(out)
+}
+
 func (l *Logger) Print(v ...interface{}) {
 	if l.empty {
 		return

matrix.go

@@ -2,9 +2,11 @@ package main
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"net/http"
 	"strings"
+	"sync"
 	"time"
 
 	"github.com/gomarkdown/markdown"
@@ -15,18 +17,23 @@ import (
 	"maunium.net/go/mautrix/id"
 )
 
+var (
+	DEVICE_ID = id.DeviceID("jfa-go")
+)
+
 type MatrixDaemon struct {
 	Stopped      bool
-	ShutdownChannel chan string
 	bot          *mautrix.Client
 	userID       id.UserID
+	homeserver   string
 	tokens       map[string]UnverifiedUser // Map of tokens to users
 	languages    map[id.RoomID]string      // Map of roomIDs to language codes
 	Encryption   bool
-	isEncrypted     map[id.RoomID]bool
+	crypto       *Crypto
-	crypto          Crypto
 	app          *appContext
 	start        int64
+	cancellation sync.WaitGroup
+	cancel       context.CancelFunc
 }
 
 type UnverifiedUser struct {
@@ -57,23 +64,33 @@ var matrixFilter = mautrix.Filter{
 	},
 }
 
+func (d *MatrixDaemon) renderUserID(uid id.UserID) id.UserID {
+	if uid[0] != '@' {
+		uid = "@" + uid
+	}
+	if !strings.ContainsRune(string(uid), ':') {
+		uid = id.UserID(string(uid) + ":" + d.homeserver)
+	}
+	return uid
+}
+
 func newMatrixDaemon(app *appContext) (d *MatrixDaemon, err error) {
 	matrix := app.config.Section("matrix")
-	homeserver := matrix.Key("homeserver").String()
 	token := matrix.Key("token").String()
 	d = &MatrixDaemon{
-		ShutdownChannel: make(chan string),
 		userID:     id.UserID(matrix.Key("user_id").String()),
+		homeserver: matrix.Key("homeserver").String(),
 		tokens:     map[string]UnverifiedUser{},
 		languages:  map[id.RoomID]string{},
-		isEncrypted:     map[id.RoomID]bool{},
 		app:        app,
 		start:      time.Now().UnixNano() / 1e6,
 	}
-	d.bot, err = mautrix.NewClient(homeserver, d.userID, token)
+	d.userID = d.renderUserID(d.userID)
+	d.bot, err = mautrix.NewClient(d.homeserver, d.userID, token)
 	if err != nil {
 		return
 	}
+	d.bot.DeviceID = DEVICE_ID
 	// resp, err := d.bot.CreateFilter(&matrixFilter)
 	// if err != nil {
 	// 	return
@@ -83,7 +100,6 @@ func newMatrixDaemon(app *appContext) (d *MatrixDaemon, err error) {
 		if user.Lang != "" {
 			d.languages[id.RoomID(user.RoomID)] = user.Lang
 		}
-		d.isEncrypted[id.RoomID(user.RoomID)] = user.Encrypted
 	}
 	err = InitMatrixCrypto(d)
 	return
@@ -102,7 +118,7 @@ func (d *MatrixDaemon) generateAccessToken(homeserver, username, password string
 			User: username,
 		},
 		Password: password,
-		DeviceID: id.DeviceID("jfa-go-" + commit),
+		DeviceID: DEVICE_ID,
 	}
 	bot, err := mautrix.NewClient(homeserver, id.UserID(username), "")
 	if err != nil {
@@ -116,22 +132,25 @@ func (d *MatrixDaemon) generateAccessToken(homeserver, username, password string
 }
 
 func (d *MatrixDaemon) run() {
-	startTime := d.start
-	d.app.info.Println(lm.StartDaemon, lm.Matrix)
 	syncer := d.bot.Syncer.(*mautrix.DefaultSyncer)
-	HandleSyncerCrypto(startTime, d, syncer)
 	syncer.OnEventType(event.EventMessage, d.handleMessage)
 
-	if err := d.bot.Sync(); err != nil {
+	d.app.info.Printf(lm.StartDaemon, lm.Matrix)
+
+	var syncCtx context.Context
+	syncCtx, d.cancel = context.WithCancel(context.Background())
+	d.cancellation.Add(1)
+
+	if err := d.bot.SyncWithContext(syncCtx); err != nil && !errors.Is(err, context.Canceled) {
 		d.app.err.Printf(lm.FailedSyncMatrix, err)
 	}
+	d.cancellation.Done()
 }
 
 func (d *MatrixDaemon) Shutdown() {
-	CryptoShutdown(d)
+	d.cancel()
-	d.bot.StopSync()
+	d.cancellation.Wait()
 	d.Stopped = true
-	close(d.ShutdownChannel)
 }
 
 func (d *MatrixDaemon) handleMessage(ctx context.Context, evt *event.Event) {
@@ -184,7 +203,7 @@ func (d *MatrixDaemon) commandLang(evt *event.Event, code, lang string) {
 	}
 }
 
-func (d *MatrixDaemon) CreateRoom(userID string) (roomID id.RoomID, encrypted bool, err error) {
+func (d *MatrixDaemon) CreateRoom(userID string) (roomID id.RoomID, err error) {
 	var room *mautrix.RespCreateRoom
 	room, err = d.bot.CreateRoom(context.TODO(), &mautrix.ReqCreateRoom{
 		Visibility: "private",
@@ -195,13 +214,14 @@ func (d *MatrixDaemon) CreateRoom(userID string) (roomID id.RoomID, encrypted bo
 	if err != nil {
 		return
 	}
-	encrypted = EncryptRoom(d, room, id.UserID(userID))
+	// encrypted = EncryptRoom(d, room, id.UserID(userID))
 	roomID = room.RoomID
+	err = EncryptRoom(d, roomID)
 	return
 }
 
 func (d *MatrixDaemon) SendStart(userID string) (ok bool) {
-	roomID, encrypted, err := d.CreateRoom(userID)
+	roomID, err := d.CreateRoom(userID)
 	if err != nil {
 		d.app.err.Printf(lm.FailedCreateMatrixRoom, userID, err)
 		return
@@ -214,7 +234,6 @@ func (d *MatrixDaemon) SendStart(userID string) (ok bool) {
 			RoomID: string(roomID),
 			UserID: userID,
 			Lang:   lang,
-			Encrypted: encrypted,
 		},
 	}
 	err = d.sendToRoom(
@@ -234,12 +253,13 @@ func (d *MatrixDaemon) SendStart(userID string) (ok bool) {
 }
 
 func (d *MatrixDaemon) sendToRoom(content *event.MessageEventContent, roomID id.RoomID) (err error) {
-	if encrypted, ok := d.isEncrypted[roomID]; ok && encrypted {
+	return d.send(content, roomID)
+	/*if encrypted, ok := d.isEncrypted[roomID]; ok && encrypted {
 		err = SendEncrypted(d, content, roomID)
 	} else {
 		_, err = d.bot.SendMessageEvent(context.TODO(), roomID, event.EventMessage, content, mautrix.ReqSendEvent{})
 	}
-	return
+	return*/
 }
 
 func (d *MatrixDaemon) send(content *event.MessageEventContent, roomID id.RoomID) (err error) {

matrix_crypto.go

@@ -4,224 +4,54 @@
 package main
 
 import (
-	"fmt"
+	"context"
-	"strings"
 
-	lm "github.com/hrfee/jfa-go/logmessages"
+	_ "github.com/mattn/go-sqlite3"
-	"maunium.net/go/mautrix"
+	"maunium.net/go/mautrix/crypto/cryptohelper"
-	"maunium.net/go/mautrix/crypto"
 	"maunium.net/go/mautrix/event"
 	"maunium.net/go/mautrix/id"
 )
 
 type Crypto struct {
-	cryptoStore *crypto.GobStore
+	helper *cryptohelper.CryptoHelper
-	olm         *crypto.OlmMachine
 }
 
 func MatrixE2EE() bool { return true }
 
-type stateStore struct {
+func InitMatrixCrypto(d *MatrixDaemon) error {
-	isEncrypted *map[id.RoomID]bool
-}
-
-func (m *stateStore) IsEncrypted(roomID id.RoomID) bool {
-	// encrypted, ok := (*m.isEncrypted)[roomID]
-	// return ok && encrypted
-	return true
-}
-
-func (m *stateStore) GetEncryptionEvent(roomID id.RoomID) *event.EncryptionEventContent {
-	return &event.EncryptionEventContent{
-		Algorithm:              id.AlgorithmMegolmV1,
-		RotationPeriodMillis:   7 * 24 * 60 * 60 * 1000,
-		RotationPeriodMessages: 100,
-	}
-}
-
-// Users are assumed to only have one common channel with the bot, so we can stub this out.
-func (m *stateStore) FindSharedRooms(userID id.UserID) []id.RoomID {
-	// for _, user := range m.app.storage.matrix {
-	// 	if id.UserID(user.UserID) == userID {
-	// 		return []id.RoomID{id.RoomID(user.RoomID)}
-	// 	}
-	// }
-	return []id.RoomID{}
-}
-
-func (d *MatrixDaemon) getUserIDs(roomID id.RoomID) (list []id.UserID, err error) {
-	members, err := d.bot.JoinedMembers(roomID)
-	if err != nil {
-		return
-	}
-	list = make([]id.UserID, len(members.Joined))
-	i := 0
-	for id := range members.Joined {
-		list[i] = id
-		i++
-	}
-	return
-}
-
-type olmLogger struct {
-	app *appContext
-}
-
-func (o olmLogger) Error(message string, args ...interface{}) {
-	o.app.err.Printf(lm.MatrixOlmLog, fmt.Sprintf(message, args))
-}
-
-func (o olmLogger) Warn(message string, args ...interface{}) {
-	o.app.info.Printf(lm.MatrixOlmLog, fmt.Sprintf(message, args))
-}
-
-func (o olmLogger) Debug(message string, args ...interface{}) {
-	o.app.debug.Printf(lm.MatrixOlmLog, fmt.Sprintf(message, args))
-}
-
-func (o olmLogger) Trace(message string, args ...interface{}) {
-	if strings.HasPrefix(message, "Got membership state event") {
-		return
-	}
-	o.app.debug.Printf(lm.MatrixOlmTracelog, fmt.Sprintf(message, args))
-}
-
-func InitMatrixCrypto(d *MatrixDaemon) (err error) {
 	d.Encryption = d.app.config.Section("matrix").Key("encryption").MustBool(false)
 	if !d.Encryption {
-		return
+		// return fmt.Errorf("encryption disabled")
-	}
+		return nil
-	for _, user := range d.app.storage.matrix {
-		d.isEncrypted[id.RoomID(user.RoomID)] = user.Encrypted
 	}
+
 	dbPath := d.app.config.Section("files").Key("matrix_sql").String()
-	// If the db is maintained after restart, element reports "The secure channel with the sender was corrupted" when sending a message from the bot.
+	var err error
-	// This obviously isn't right, but it seems to work.
+	d.crypto = &Crypto{}
-	// Since its not really used anyway, just use the deprecated GobStore. This reduces cgo usage anyway.
+	d.crypto.helper, err = cryptohelper.NewCryptoHelper(d.bot, []byte("jfa-go"), dbPath)
-	var cryptoStore *crypto.GobStore
-	cryptoStore, err = crypto.NewGobStore(dbPath)
-	// d.db, err = sql.Open("sqlite3", dbPath)
 	if err != nil {
-		return
+		return err
-	}
-	olmLog := &olmLogger{d.app}
-	// deviceID := "jfa-go" + commit
-	// cryptoStore := crypto.NewSQLCryptoStore(d.db, "sqlite3", string(d.userID)+deviceID, id.DeviceID(deviceID), []byte("jfa-go"), olmLog)
-	// err = cryptoStore.CreateTables()
-	// if err != nil {
-	// 	return
-	// }
-	olm := crypto.NewOlmMachine(d.bot, olmLog, cryptoStore, &stateStore{&d.isEncrypted})
-	olm.AllowUnverifiedDevices = true
-	err = olm.Load()
-	if err != nil {
-		return
-	}
-	d.crypto = Crypto{
-		cryptoStore: cryptoStore,
-		olm:         olm,
-	}
-	return
 	}
 
-func HandleSyncerCrypto(startTime int64, d *MatrixDaemon, syncer *mautrix.DefaultSyncer) {
+	err = d.crypto.helper.Init(context.TODO())
+	if err != nil {
+		return err
+	}
+
+	d.bot.Crypto = d.crypto.helper
+
+	d.Encryption = true
+	return nil
+}
+
+func EncryptRoom(d *MatrixDaemon, roomID id.RoomID) error {
 	if !d.Encryption {
-		return
+		return nil
 	}
-	syncer.OnSync(func(resp *mautrix.RespSync, since string) bool {
+	_, err := d.bot.SendStateEvent(context.TODO(), roomID, event.StateEncryption, "", event.EncryptionEventContent{
-		d.crypto.olm.ProcessSyncResponse(resp, since)
-		return true
-	})
-	syncer.OnEventType(event.StateMember, func(source mautrix.EventSource, evt *event.Event) {
-		d.crypto.olm.HandleMemberEvent(evt)
-		// if evt.Content.AsMember().Membership != event.MembershipJoin {
-		// 	return
-		// }
-		// userIDs, err := d.getUserIDs(evt.RoomID)
-		// if err != nil || len(userIDs) < 2 {
-		// 	fmt.Println("FS", err)
-		// 	return
-		// }
-		// err = d.crypto.olm.ShareGroupSession(evt.RoomID, userIDs)
-		// if err != nil {
-		// 	fmt.Println("FS", err)
-		// 	return
-		// }
-	})
-	syncer.OnEventType(event.EventEncrypted, func(source mautrix.EventSource, evt *event.Event) {
-		if evt.Timestamp < startTime {
-			return
-		}
-		decrypted, err := d.crypto.olm.DecryptMegolmEvent(evt)
-		// if strings.Contains(err.Error(), crypto.NoSessionFound.Error()) {
-		// 	d.app.err.Printf("Failed to decrypt Matrix message: no session found")
-		// 	return
-		// }
-		if err != nil {
-			d.app.err.Printf(lm.FailedDecryptMatrixMessage, err)
-			return
-		}
-		d.handleMessage(source, decrypted)
-	})
-}
-
-func CryptoShutdown(d *MatrixDaemon) {
-	if d.Encryption {
-		d.crypto.olm.FlushStore()
-	}
-}
-
-func EncryptRoom(d *MatrixDaemon, room *mautrix.RespCreateRoom, userID id.UserID) (encrypted bool) {
-	if !d.Encryption {
-		return
-	}
-	_, err := d.bot.SendStateEvent(room.RoomID, event.StateEncryption, "", &event.EncryptionEventContent{
 		Algorithm:              id.AlgorithmMegolmV1,
 		RotationPeriodMillis:   7 * 24 * 60 * 60 * 1000,
 		RotationPeriodMessages: 100,
 	})
-	if err == nil {
+	return err
-		encrypted = true
-	} else {
-		d.app.debug.Printf(lm.FailedEnableMatrixEncryption, room.RoomID, err)
-		return
-	}
-	d.isEncrypted[room.RoomID] = encrypted
-	var userIDs []id.UserID
-	userIDs, err = d.getUserIDs(room.RoomID)
-	if err != nil {
-		return
-	}
-	userIDs = append(userIDs, userID)
-	return
-}
-
-func SendEncrypted(d *MatrixDaemon, content *event.MessageEventContent, roomID id.RoomID) (err error) {
-	if !d.Encryption {
-		err = d.send(content, roomID)
-		return
-	}
-	var encrypted *event.EncryptedEventContent
-	encrypted, err = d.crypto.olm.EncryptMegolmEvent(roomID, event.EventMessage, content)
-	if err == crypto.SessionExpired || err == crypto.SessionNotShared || err == crypto.NoGroupSession {
-		// err = d.crypto.olm.ShareGroupSession(id.RoomID(user.RoomID), []id.UserID{id.UserID(user.UserID), d.userID})
-		var userIDs []id.UserID
-		userIDs, err = d.getUserIDs(roomID)
-		if err != nil {
-			return
-		}
-		err = d.crypto.olm.ShareGroupSession(roomID, userIDs)
-		if err != nil {
-			return
-		}
-		encrypted, err = d.crypto.olm.EncryptMegolmEvent(roomID, event.EventMessage, content)
-	}
-	if err != nil {
-		return
-	}
-	_, err = d.bot.SendMessageEvent(roomID, event.EventEncrypted, &event.Content{Parsed: encrypted})
-	if err != nil {
-		return
-	}
-	return
 }

matrix_nocrypto.go

@@ -1,12 +1,9 @@
+//go:build !e2ee
 // +build !e2ee
 
 package main
 
-import (
+import "maunium.net/go/mautrix/id"
-	"maunium.net/go/mautrix"
-	"maunium.net/go/mautrix/event"
-	"maunium.net/go/mautrix/id"
-)
 
 type Crypto struct{}
 
@@ -17,19 +14,4 @@ func InitMatrixCrypto(d *MatrixDaemon) (err error) {
 	return
 }
 
-func HandleSyncerCrypto(startTime int64, d *MatrixDaemon, syncer *mautrix.DefaultSyncer) {
+func EncryptRoom(d *MatrixDaemon, roomID id.RoomID) error { return nil }
-	return
-}
-
-func CryptoShutdown(d *MatrixDaemon) {
-	return
-}
-
-func EncryptRoom(d *MatrixDaemon, room *mautrix.RespCreateRoom, userID id.UserID) (encrypted bool) {
-	return
-}
-
-func SendEncrypted(d *MatrixDaemon, content *event.MessageEventContent, roomID id.RoomID) (err error) {
-	err = d.send(content, roomID)
-	return
-}

pwreset.go

@@ -40,7 +40,7 @@ func (app *appContext) GenResetLink(pin string) (string, error) {
 }
 
 func (app *appContext) StartPWR() {
-	app.info.Println(lm.StartDaemon, "PWR")
+	app.info.Printf(lm.StartDaemon, "PWR")
 	path := app.config.Section("password_resets").Key("watch_directory").String()
 	if _, err := os.Stat(path); os.IsNotExist(err) {
 		app.err.Printf(lm.FailedStartDaemon, "PWR", fmt.Sprintf(lm.PathNotFound, path))

storage.go

@@ -650,7 +650,6 @@ type TelegramUser struct {
 
 type MatrixUser struct {
 	RoomID     string
-	Encrypted  bool
 	UserID     string
 	Lang       string
 	Contact    bool