activity: add limiting settings

limit to keeping n most recent logs, and/or logs younger than {n} days
in settings > Activity Log.

config.go

@@ -78,6 +78,9 @@ func (app *appContext) loadConfig() error {
 	app.MustSetValue("smtp", "cert_validation", "true")
 	app.MustSetValue("smtp", "auth_type", "4")
 
+	app.MustSetValue("activity_log", "keep_n_records", "1000")
+	app.MustSetValue("activity_log", "delete_after_days", "90")
+
 	sc := app.config.Section("discord").Key("start_command").MustString("start")
 	app.config.Section("discord").Key("start_command").SetValue(strings.TrimPrefix(strings.TrimPrefix(sc, "/"), "!"))
 

config/config-base.json

@@ -515,6 +515,31 @@
                 }
             }
         },
+        "activity_log": {
+            "order": [],
+            "meta": {
+                "name": "Activity Log",
+                "description": "Settings for data retention of the activity log."
+            },
+            "settings": {
+                "keep_n_records": {
+                    "name": "Number of records to keep",
+                    "required": false,
+                    "requires_restart": true,
+                    "type": "number",
+                    "value": 1000,
+                    "description": "How many of the most recent activities to keep. Set to 0 to disable."
+                },
+                "delete_after_days": {
+                    "name": "Delete activities older than (days):",
+                    "required": false,
+                    "requires_restart": true,
+                    "type": "number",
+                    "value": 90,
+                    "description": "If an activity was created this many days ago, it will be deleted. Set to 0 to disable."
+                }
+            }
+        },
         "captcha": {
             "order": [],
             "meta": {

daemon.go

@@ -3,7 +3,9 @@ package main
 import (
 	"time"
 
+	"github.com/dgraph-io/badger/v3"
 	"github.com/hrfee/mediabrowser"
+	"github.com/timshannon/badgerhold/v4"
 )
 
 // clearEmails removes stored emails for users which no longer exist.
@@ -72,6 +74,37 @@ func (app *appContext) clearTelegram() {
 	}
 }
 
+func (app *appContext) clearActivities() {
+	app.debug.Println("Husekeeping: Cleaning up Activity log...")
+	keepCount := app.config.Section("activity_log").Key("keep_n_records").MustInt(1000)
+	maxAgeDays := app.config.Section("activity_log").Key("delete_after_days").MustInt(90)
+	minAge := time.Now().AddDate(0, 0, -maxAgeDays)
+	err := error(nil)
+	errorSource := 0
+	if maxAgeDays != 0 {
+		err = app.storage.db.DeleteMatching(&Activity{}, badgerhold.Where("Time").Lt(minAge))
+	}
+	if err == nil && keepCount != 0 {
+		// app.debug.Printf("Keeping %d records", keepCount)
+		err = app.storage.db.DeleteMatching(&Activity{}, (&badgerhold.Query{}).Reverse().SortBy("Time").Skip(keepCount))
+		if err != nil {
+			errorSource = 1
+		}
+	}
+	if err == badger.ErrTxnTooBig {
+		app.debug.Printf("Activities: Delete txn was too big, doing it manually.")
+		list := []Activity{}
+		if errorSource == 0 {
+			app.storage.db.Find(&list, badgerhold.Where("Time").Lt(minAge))
+		} else {
+			app.storage.db.Find(&list, (&badgerhold.Query{}).Reverse().SortBy("Time").Skip(keepCount))
+		}
+		for _, record := range list {
+			app.storage.DeleteActivityKey(record.ID)
+		}
+	}
+}
+
 // https://bbengfort.github.io/snippets/2016/06/26/background-work-goroutines-timer.html THANKS
 
 type housekeepingDaemon struct {
@@ -91,10 +124,13 @@ func newInviteDaemon(interval time.Duration, app *appContext) *housekeepingDaemo
 		period:          interval,
 		app:             app,
 	}
-	daemon.jobs = []func(app *appContext){func(app *appContext) {
-		app.debug.Println("Housekeeping: Checking for expired invites")
-		app.checkInvites()
-	}}
+	daemon.jobs = []func(app *appContext){
+		func(app *appContext) {
+			app.debug.Println("Housekeeping: Checking for expired invites")
+			app.checkInvites()
+		},
+		func(app *appContext) { app.clearActivities() },
+	}
 
 	clearEmail := app.config.Section("email").Key("require_unique").MustBool(false)
 	clearDiscord := app.config.Section("discord").Key("require_unique").MustBool(false)

ts/modules/activity.ts

@@ -704,7 +704,7 @@ export class activityList {
                 if (this._search.inSearch && !this._lastPage) this._loadAllButton.classList.remove("unfocused");
                 else this._loadAllButton.classList.add("unfocused");
                 
-                if (visibleCount < 10) {
+                if (visibleCount < 10 || loadAll) {
                     if (!newItems || this._prevResultCount != visibleCount || (visibleCount == 0 && !this._lastPage) || loadAll) this.loadMore(() => {}, loadAll);
                 }
                 this._prevResultCount = visibleCount;