activity: add limiting settings

limit to keeping n most recent logs, and/or logs younger than {n} days
in settings > Activity Log.

api-activities.go

@@ -168,3 +168,19 @@ func (app *appContext) DeleteActivity(gc *gin.Context) {
 	app.storage.DeleteActivityKey(gc.Param("id"))
 	respondBool(200, true, gc)
 }
+
+// @Summary Returns the total number of activities stored in the database.
+// @Produce json
+// @Success 200 {object} GetActivityCountDTO
+// @Router /activity/count [get]
+// @Security Bearer
+// @tags Activity
+func (app *appContext) GetActivityCount(gc *gin.Context) {
+	resp := GetActivityCountDTO{}
+	var err error
+	resp.Count, err = app.storage.db.Count(&Activity{}, &badgerhold.Query{})
+	if err != nil {
+		resp.Count = 0
+	}
+	gc.JSON(200, resp)
+}

config.go

@@ -78,6 +78,9 @@ func (app *appContext) loadConfig() error {
 	app.MustSetValue("smtp", "cert_validation", "true")
 	app.MustSetValue("smtp", "auth_type", "4")
 
+	app.MustSetValue("activity_log", "keep_n_records", "1000")
+	app.MustSetValue("activity_log", "delete_after_days", "90")
+
 	sc := app.config.Section("discord").Key("start_command").MustString("start")
 	app.config.Section("discord").Key("start_command").SetValue(strings.TrimPrefix(strings.TrimPrefix(sc, "/"), "!"))
 

config/config-base.json

@@ -515,6 +515,31 @@
                 }
             }
         },
+        "activity_log": {
+            "order": [],
+            "meta": {
+                "name": "Activity Log",
+                "description": "Settings for data retention of the activity log."
+            },
+            "settings": {
+                "keep_n_records": {
+                    "name": "Number of records to keep",
+                    "required": false,
+                    "requires_restart": true,
+                    "type": "number",
+                    "value": 1000,
+                    "description": "How many of the most recent activities to keep. Set to 0 to disable."
+                },
+                "delete_after_days": {
+                    "name": "Delete activities older than (days):",
+                    "required": false,
+                    "requires_restart": true,
+                    "type": "number",
+                    "value": 90,
+                    "description": "If an activity was created this many days ago, it will be deleted. Set to 0 to disable."
+                }
+            }
+        },
         "captcha": {
             "order": [],
             "meta": {

daemon.go

@@ -3,7 +3,9 @@ package main
 import (
 	"time"
 
+	"github.com/dgraph-io/badger/v3"
 	"github.com/hrfee/mediabrowser"
+	"github.com/timshannon/badgerhold/v4"
 )
 
 // clearEmails removes stored emails for users which no longer exist.
@@ -72,6 +74,37 @@ func (app *appContext) clearTelegram() {
 	}
 }
 
+func (app *appContext) clearActivities() {
+	app.debug.Println("Husekeeping: Cleaning up Activity log...")
+	keepCount := app.config.Section("activity_log").Key("keep_n_records").MustInt(1000)
+	maxAgeDays := app.config.Section("activity_log").Key("delete_after_days").MustInt(90)
+	minAge := time.Now().AddDate(0, 0, -maxAgeDays)
+	err := error(nil)
+	errorSource := 0
+	if maxAgeDays != 0 {
+		err = app.storage.db.DeleteMatching(&Activity{}, badgerhold.Where("Time").Lt(minAge))
+	}
+	if err == nil && keepCount != 0 {
+		// app.debug.Printf("Keeping %d records", keepCount)
+		err = app.storage.db.DeleteMatching(&Activity{}, (&badgerhold.Query{}).Reverse().SortBy("Time").Skip(keepCount))
+		if err != nil {
+			errorSource = 1
+		}
+	}
+	if err == badger.ErrTxnTooBig {
+		app.debug.Printf("Activities: Delete txn was too big, doing it manually.")
+		list := []Activity{}
+		if errorSource == 0 {
+			app.storage.db.Find(&list, badgerhold.Where("Time").Lt(minAge))
+		} else {
+			app.storage.db.Find(&list, (&badgerhold.Query{}).Reverse().SortBy("Time").Skip(keepCount))
+		}
+		for _, record := range list {
+			app.storage.DeleteActivityKey(record.ID)
+		}
+	}
+}
+
 // https://bbengfort.github.io/snippets/2016/06/26/background-work-goroutines-timer.html THANKS
 
 type housekeepingDaemon struct {
@@ -91,10 +124,13 @@ func newInviteDaemon(interval time.Duration, app *appContext) *housekeepingDaemo
 		period:          interval,
 		app:             app,
 	}
-	daemon.jobs = []func(app *appContext){func(app *appContext) {
-		app.debug.Println("Housekeeping: Checking for expired invites")
-		app.checkInvites()
-	}}
+	daemon.jobs = []func(app *appContext){
+		func(app *appContext) {
+			app.debug.Println("Housekeeping: Checking for expired invites")
+			app.checkInvites()
+		},
+		func(app *appContext) { app.clearActivities() },
+	}
 
 	clearEmail := app.config.Section("email").Key("require_unique").MustBool(false)
 	clearDiscord := app.config.Section("discord").Key("require_unique").MustBool(false)

html/admin.html

@@ -737,7 +737,14 @@
                         <span class="button ~neutral @low center ml-[-2.64rem] rounded-s-none activity-search-clear" aria-label="{{ .strings.clearSearch }}" text="{{ .strings.clearSearch }}"><i class="ri-close-line"></i></span>
                         <button class="button ~info @low ml-2" id="activity-refresh" aria-label="{{ .strings.refresh }}" disabled><i class="ri-refresh-line"></i></button>
                     </div>
-                    <div class="supra py-1 sm hidden" id="activity-search-options-header">{{ .strings.searchOptions }}</div>
+                    <div class="flex flex-row justify-between py-2">
+                        <div class="supra sm hidden" id="activity-search-options-header">{{ .strings.searchOptions }}</div>
+                        <div class="supra sm">
+                            <span id="activity-total-records" class="mx-2"></span>
+                            <span id="activity-loaded-records" class="mx-2"></span>
+                            <span id="activity-shown-records" class="mx-2"></span>
+                        </div>
+                    </div>
                     <div class="row -mx-2 mb-2">
                         <button type="button" class="button ~neutral @low center mx-2 hidden"><span id="activity-sort-by-field"></span> <i class="ri-close-line ml-2 text-2xl"></i></button>
                         <span id="activity-filter-area"></span>

lang/admin/en-us.json

@@ -172,7 +172,10 @@
         "inviteDeletedFilter": "Invite Deleted/Expired",
         "loadMore": "Load More",
         "loadAll": "Load All",
-        "noMoreResults": "No more results."
+        "noMoreResults": "No more results.",
+        "totalRecords": "{n} Total Records",
+        "loadedRecords": "{n} Loaded",
+        "shownRecords": "{n} Shown"
     },
     "notifications": {
         "changedEmailAddress": "Changed email address of {n}.",

models.go

@@ -455,3 +455,7 @@ type GetActivitiesRespDTO struct {
 	Activities []ActivityDTO `json:"activities"`
 	LastPage   bool          `json:"last_page"`
 }
+
+type GetActivityCountDTO struct {
+	Count uint64 `json:"count"`
+}

router.go

@@ -237,6 +237,7 @@ func (app *appContext) loadRoutes(router *gin.Engine) {
 
 		api.POST(p+"/activity", app.GetActivities)
 		api.DELETE(p+"/activity/:id", app.DeleteActivity)
+		api.GET(p+"/activity/count", app.GetActivityCount)
 
 		if userPageEnabled {
 			user.GET("/details", app.MyDetails)

ts/modules/activity.ts

@@ -1,4 +1,4 @@
-import { _post, _delete, toDateString, addLoader, removeLoader } from "../modules/common.js";
+import { _get, _post, _delete, toDateString, addLoader, removeLoader } from "../modules/common.js";
 import { Search, SearchConfiguration, QueryType, SearchableItem } from "../modules/search.js";
 import { accountURLEvent } from "../modules/accounts.js";
 import { inviteURLEvent } from "../modules/invites.js";
@@ -357,6 +357,32 @@ export class activityList {
     private _keepSearchingDescription = document.getElementById("activity-keep-searching-description");
     private _keepSearchingButton = document.getElementById("activity-keep-searching");
 
+    private _totalRecords = document.getElementById("activity-total-records");
+    private _loadedRecords = document.getElementById("activity-loaded-records");
+    private _shownRecords = document.getElementById("activity-shown-records");
+
+    private _total: number;
+    private _loaded: number;
+    private _shown: number;
+
+    get total(): number { return this._total; }
+    set total(v: number) {
+        this._total = v;
+        this._totalRecords.textContent = window.lang.var("strings", "totalRecords", `${v}`);
+    }
+    
+    get loaded(): number { return this._loaded; }
+    set loaded(v: number) {
+        this._loaded = v;
+        this._loadedRecords.textContent = window.lang.var("strings", "loadedRecords", `${v}`);
+    }
+    
+    get shown(): number { return this._shown; }
+    set shown(v: number) {
+        this._shown = v;
+        this._shownRecords.textContent = window.lang.var("strings", "shownRecords", `${v}`);
+    }
+
     private _search: Search;
     private _ascending: boolean;
     private _hasLoaded: boolean;
@@ -383,6 +409,11 @@ export class activityList {
         this._loadMoreButton.disabled = false;
         this._loadAllButton.classList.remove("unfocused");
         this._loadAllButton.disabled = false;
+
+        this.total = 0;
+        this.loaded = 0;
+        this.shown = 0;
+
         // this._page = 0;
         let limit = 10;
         if (this._page != 0) {
@@ -396,6 +427,11 @@ export class activityList {
             "ascending": this.ascending
         }
 
+        _get("/activity/count", null, (req: XMLHttpRequest) => {
+            if (req.readyState != 4 || req.status != 200) return;
+            this.total = req.response["count"] as number;
+        });
+
         _post("/activity", send, (req: XMLHttpRequest) => {
             if (req.readyState != 4) return;
             if (req.status != 200) {
@@ -420,6 +456,8 @@ export class activityList {
             this._search.items = this._activities;
             this._search.ordering = this._ordering;
 
+            this.loaded = this._ordering.length;
+
             if (this._search.inSearch) {
                 this._search.onSearchBoxChange(true);
                 this._loadAllButton.classList.remove("unfocused");
@@ -476,6 +514,8 @@ export class activityList {
             // this._search.items = this._activities;
             // this._search.ordering = this._ordering;
 
+            this.loaded = this._ordering.length;
+
             if (this._search.inSearch || loadAll) {
                 if (this._lastPage) {
                     loadAll = false;
@@ -659,10 +699,12 @@ export class activityList {
             filterList: document.getElementById("activity-filter-list"),
             // notFoundCallback: this._notFoundCallback,
             onSearchCallback: (visibleCount: number, newItems: boolean, loadAll: boolean) => {
+                this.shown = visibleCount;
+
                 if (this._search.inSearch && !this._lastPage) this._loadAllButton.classList.remove("unfocused");
                 else this._loadAllButton.classList.add("unfocused");
                 
-                if (visibleCount < 10) {
+                if (visibleCount < 10 || loadAll) {
                     if (!newItems || this._prevResultCount != visibleCount || (visibleCount == 0 && !this._lastPage) || loadAll) this.loadMore(() => {}, loadAll);
                 }
                 this._prevResultCount = visibleCount;