From 1e8aea4dbd746baba82fa83c57d36ece4d2bd26b Mon Sep 17 00:00:00 2001 From: Harvey Tindall Date: Thu, 27 Feb 2025 13:06:37 +0000 Subject: [PATCH] add random item selector --- go.mod | 9 +- go.sum | 2 + main.go | 308 +++++++++++++++++++++++++++++++++++++++++++++++++++++--- 3 files changed, 302 insertions(+), 17 deletions(-) diff --git a/go.mod b/go.mod index 3f632d5..2e3a381 100644 --- a/go.mod +++ b/go.mod @@ -1,5 +1,10 @@ module git.hrfee.pw/hrfee/discogs-pricer -go 1.22.4 +go 1.23.0 -require github.com/hrfee/go-discogs v0.0.0-20240905170225-50ee0e6a98ae // indirect +toolchain go1.23.2 + +require ( + github.com/hrfee/go-discogs v0.0.0-20240905170225-50ee0e6a98ae // indirect + golang.org/x/exp v0.0.0-20250218142911-aa4b98e5adaa // indirect +) diff --git a/go.sum b/go.sum index ec9e25b..94818e7 100644 --- a/go.sum +++ b/go.sum @@ -2,3 +2,5 @@ github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/hrfee/go-discogs v0.0.0-20240905170225-50ee0e6a98ae h1:9uX6ArLOVUzryMkT3VKud4c/5vNwWEwdR6vJHHvh2tw= github.com/hrfee/go-discogs v0.0.0-20240905170225-50ee0e6a98ae/go.mod h1:Z34PS6moBg1QdybW9LkrqciRJHKdyQ9hpxvaiH8sFic= +golang.org/x/exp v0.0.0-20250218142911-aa4b98e5adaa h1:t2QcU6V556bFjYgu4L6C+6VrCPyJZ+eyRsABUPs1mz4= +golang.org/x/exp v0.0.0-20250218142911-aa4b98e5adaa/go.mod h1:BHOTPb3L19zxehTsLoJXVaTktb06DFgmdW6Wb9s8jqk= diff --git a/main.go b/main.go index d482a76..b74e8ff 100644 --- a/main.go +++ b/main.go @@ -14,9 +14,18 @@ import ( "time" "unicode" + "math/rand" + "github.com/hrfee/go-discogs" ) +const ( + CustomFieldPrice discogs.FieldID = 4 + CustomFieldShipping = 5 + CustomFieldPurchaseDate = 6 + CustomFieldFrom = 7 +) + var ( CURRENCY = "GBP" USER_AGENT = "discogs-pricer/0.0 +https://git.hrfee.pw/hrfee/discogs-pricer" @@ -27,7 +36,7 @@ var ( NotesFieldID = 4 // Test, otherwise use 3 - noPaToken = errors.New(`put ":" in ./personal_access_token and re-run.`) + errNoPersonalAccessToken = errors.New(`put ":" in ./personal_access_token and re-run`) ) type ItemHash string @@ -142,7 +151,7 @@ func (c *Client) CollectionJSON() (string, error) { return string(v), nil } -func (c *Client) WriteCSVTemplate(path string, currency string) error { +func (c *Client) WriteCSVTemplate(path string, currency string, mediaType string) error { if c.CollectionCache.Order == nil || len(c.CollectionCache.Order) == 0 { if err := c.PopulateCollectionCache(); err != nil { return err @@ -151,6 +160,17 @@ func (c *Client) WriteCSVTemplate(path string, currency string) error { return err } } + acceptTypes := []string{} + if mediaType == "cd" || mediaType == "all" { + acceptTypes = append(acceptTypes, "CD") + } + if mediaType == "cassette" || mediaType == "all" { + acceptTypes = append(acceptTypes, "Cassette") + } + if mediaType == "vinyl" || mediaType == "all" { + acceptTypes = append(acceptTypes, "Vinyl") + } + file, err := os.Create(path) if err != nil { return err @@ -161,6 +181,16 @@ func (c *Client) WriteCSVTemplate(path string, currency string) error { w.Write(HeaderRow) for _, ciHash := range c.CollectionCache.Order { l := c.CollectionCache.Items[ciHash].RowFromCollectionItem(currency) + accept := false + for _, mt := range acceptTypes { + if l[4] == mt { + accept = true + break + } + } + if !accept { + continue + } err := w.Write(l) if err != nil { err = fmt.Errorf("failed writing line \"%s\": %v", l, err) @@ -170,7 +200,124 @@ func (c *Client) WriteCSVTemplate(path string, currency string) error { return nil } -func (c *Client) ReadFromFilledCSV(path string, currency string, skipFirstNRows int) (int, error) { +func (c *Client) RandomItems(n int, mediaType string) ([]CollectionItem, error) { + if c.CollectionCache.Order == nil || len(c.CollectionCache.Order) == 0 { + if err := c.PopulateCollectionCache(); err != nil { + return []CollectionItem{}, err + } + if err := c.StoreCollectionCache(); err != nil { + return []CollectionItem{}, err + } + } + acceptTypes := []string{} + if mediaType == "cd" || mediaType == "all" { + acceptTypes = append(acceptTypes, "CD") + } + if mediaType == "cassette" || mediaType == "all" { + acceptTypes = append(acceptTypes, "Cassette") + } + if mediaType == "vinyl" || mediaType == "all" { + acceptTypes = append(acceptTypes, "Vinyl") + } + + validRows := make([]ItemHash, 0, len(c.CollectionCache.Order)) + for _, ciHash := range c.CollectionCache.Order { + l := c.CollectionCache.Items[ciHash].RowFromCollectionItem("") + accept := false + for _, mt := range acceptTypes { + if l[4] == mt { + accept = true + break + } + } + if !accept { + continue + } + validRows = append(validRows, ciHash) + } + + items := make([]CollectionItem, n) + for i := range n { + items[i] = c.CollectionCache.Items[validRows[rand.Intn(len(validRows))]] + } + + return items, nil +} + +func (c *Client) Statistics(currency string) error { + if c.CollectionCache.Order == nil || len(c.CollectionCache.Order) == 0 { + if err := c.PopulateCollectionCache(); err != nil { + return err + } + if err := c.StoreCollectionCache(); err != nil { + return err + } + } + + extremePrices := [2][2]struct { + Hash ItemHash + Price float64 + }{{{"", -1}, {"", 9999999}}, {{"", -1}, {"", 9999999}}} // Max, Min, with and without shipping + total := [2]float64{0, 0} + countWithFreeShipping := 0 // or in-person + countNonZero := 0 + + for _, ciHash := range c.CollectionCache.Order { + notes := c.CollectionCache.Items[ciHash].ParseNotes(currency) + itemPrice := notes.Price.Float() + shippingPrice := notes.ShippingPrice.Float() + totalPrice := itemPrice + shippingPrice + if !notes.Price.IsZero() || !notes.ShippingPrice.IsZero() { + countNonZero += 1 + if notes.ShippingPrice.IsZero() { + countWithFreeShipping += 1 + } + } + for i, price := range [2]float64{itemPrice, totalPrice} { + if price > extremePrices[i][0].Price { + extremePrices[i][0].Price = price + extremePrices[i][0].Hash = ciHash + } + if price < extremePrices[i][1].Price && price != 0.0 { + extremePrices[i][1].Price = price + extremePrices[i][1].Hash = ciHash + } + total[i] += price + } + } + + mean := [2]float64{total[0], total[1]} + mean[0] /= float64(countNonZero) + mean[1] /= float64(countNonZero) + + for i, title := range [2]string{"ITEM PRICE", "ITEM + SHIPPING PRICE"} { + fmt.Printf("----%s----\n", title) + for j, extremeName := range [2]string{"Maximum", "Minimum non-zero"} { + ci := c.CollectionCache.Items[extremePrices[i][j].Hash] + fmt.Printf("%s price @ %s%.2f: \"%s\"\n", extremeName, currency, extremePrices[i][j].Price, ci.String()) + } + fmt.Printf("Mean price (of all priced items): %s%.2f\n", currency, mean[i]) + fmt.Printf("Total price: %s%.2f\n", currency, total[i]) + } + + fmt.Printf("----GENERAL----\n") + pctWithFreeShipping := float64(100) * (float64(countWithFreeShipping) / float64(countNonZero)) + fmt.Printf("%d / %d (%f%%) priced items had free shipping, or were in-person pickups\n", countWithFreeShipping, countNonZero, pctWithFreeShipping) + + return nil +} + +func WaitForRateLimit(reqs, perMinute int) bool { + // fmt.Println(reqs, perMinute) + if (float64(reqs) / float64(perMinute)) > 0.8 { + fmt.Printf("waiting for rate limit\n") + time.Sleep(60 * time.Second) + return true + } + return false +} + +func (c *Client) ReadFromFilledCSV(path string, currency string, skipFirstNRows int, ratePerMinute int) (int, error) { file, err := os.Open(path) if err != nil { return 0, err @@ -178,6 +325,7 @@ func (c *Client) ReadFromFilledCSV(path string, currency string, skipFirstNRows defer file.Close() r := csv.NewReader(file) i := 0 + requestsMade := 0 for err == nil { var l []string l, err = r.Read() @@ -216,7 +364,7 @@ func (c *Client) ReadFromFilledCSV(path string, currency string, skipFirstNRows pnString := pn.ToString(currency) // fmt.Printf("For \"%s\": %s\n", l[cArtistName], pnString) - regenerated := parseNotes(pnString, "2006-01-02T15:04:05-07:00", currency) + regenerated := parseOldNotes(pnString, "2006-01-02T15:04:05-07:00", currency) pn.RoughPanickingEqual(regenerated) folderID, err := strconv.Atoi(l[cFolderID]) @@ -232,14 +380,85 @@ func (c *Client) ReadFromFilledCSV(path string, currency string, skipFirstNRows panic(err) } + // Notes err = c.c.EditFieldsInstance( c.Username, folderID, releaseID, instanceID, discogs.NotesField, - pnString, + pn.Additional, ) + requestsMade++ + if WaitForRateLimit(requestsMade, ratePerMinute) { + requestsMade = 0 + } + if err != nil { + panic(fmt.Errorf("failed on %d-%d-%d \"%s\": %v", folderID, releaseID, instanceID, l[cArtistName], err)) + } + // Price + err = c.c.EditFieldsInstance( + c.Username, + folderID, + releaseID, + instanceID, + CustomFieldPrice, + pn.Price.String(), + ) + requestsMade++ + if WaitForRateLimit(requestsMade, ratePerMinute) { + fmt.Printf("reset rate limit counter at %d\n", requestsMade) + requestsMade = 0 + } + if err != nil { + panic(fmt.Errorf("failed on %d-%d-%d \"%s\": %v", folderID, releaseID, instanceID, l[cArtistName], err)) + } + if !pn.ShippingPrice.IsZero() { + err = c.c.EditFieldsInstance( + c.Username, + folderID, + releaseID, + instanceID, + CustomFieldShipping, + pn.ShippingPrice.String(), + ) + requestsMade++ + if WaitForRateLimit(requestsMade, ratePerMinute) { + requestsMade = 0 + } + if err != nil { + panic(fmt.Errorf("failed on %d-%d-%d \"%s\": %v", folderID, releaseID, instanceID, l[cArtistName], err)) + } + } + // Purchase Date + err = c.c.EditFieldsInstance( + c.Username, + folderID, + releaseID, + instanceID, + CustomFieldPurchaseDate, + pn.PurchaseDate.Format("2006-01-02"), + ) + requestsMade++ + if WaitForRateLimit(requestsMade, ratePerMinute) { + requestsMade = 0 + } + if err != nil { + panic(fmt.Errorf("failed on %d-%d-%d \"%s\": %v", folderID, releaseID, instanceID, l[cArtistName], err)) + } + // From/Seller + err = c.c.EditFieldsInstance( + c.Username, + folderID, + releaseID, + instanceID, + CustomFieldFrom, + pn.Seller, + ) + requestsMade++ + if WaitForRateLimit(requestsMade, ratePerMinute) { + requestsMade = 0 + } if err != nil { panic(fmt.Errorf("failed on %d-%d-%d \"%s\": %v", folderID, releaseID, instanceID, l[cArtistName], err)) } @@ -290,10 +509,10 @@ func ParseTime(t string) (time.Time, error) { return time.Parse("2006-01-02T15:04:05-07:00", t) } -func (ci CollectionItem) RawNotes() string { +func (ci CollectionItem) NotesByField(f discogs.FieldID) string { notes := "" for _, n := range ci.Notes { - if n.FieldID == discogs.NotesField { + if n.FieldID == f { notes = n.Value } } @@ -333,6 +552,10 @@ func (pp ParsedPrice) String() string { return fmt.Sprintf("%s%d.%02d", pp.Currency, pp.Major, pp.Minor) } +func (pp ParsedPrice) Float() float64 { + return float64(pp.Major) + (float64(pp.Minor) / float64(100)) +} + func (pp *ParsedPrice) FromFloat(p float64) { pp.Major = int(p) pp.Minor = int((float64(p) - float64(int(p))) * float64(100)) @@ -341,6 +564,9 @@ func (pp *ParsedPrice) FromFloat(p float64) { func (pp ParsedPrice) IsZero() bool { return pp.Major == 0 && pp.Minor == 0 } func parsePrice(p string, currency string) ParsedPrice { + if p == "" { + return ParsedPrice{currency, 0, 0} + } parts := strings.SplitN(p, ".", 2) major := parts[0] existingCurrency := "" @@ -382,11 +608,34 @@ func (pn ParsedNotes) ToString(currency string) string { } func (ci CollectionItem) ParseNotes(currency string) ParsedNotes { - notes := ci.RawNotes() - return parseNotes(notes, ci.DateAdded, currency) + return parseNotes( + ci.NotesByField(discogs.NotesField), + ci.NotesByField(CustomFieldPrice), + ci.NotesByField(CustomFieldShipping), + ci.NotesByField(CustomFieldPurchaseDate), + ci.NotesByField(CustomFieldFrom), + currency, + ) } -func parseNotes(notes string, dateAdded string, currency string) ParsedNotes { +func parseNotes(notes string, price string, shipping string, purchaseDate string, from string, currency string) ParsedNotes { + pn := ParsedNotes{} + pn.Price = parsePrice(price, currency) + pn.ShippingPrice = parsePrice(shipping, currency) + var err error + pn.PurchaseDate, err = time.Parse("2006-01-02", purchaseDate) + if err != nil || purchaseDate == "" || pn.PurchaseDate.IsZero() { + if purchaseDate != "" { + fmt.Printf("date error: %v\n", err) + } + pn.PurchaseDate, _ = ParseTime(purchaseDate) + } + + pn.Seller = from + return pn +} + +func parseOldNotes(notes string, dateAdded string, currency string) ParsedNotes { pn := ParsedNotes{} dateString := "" priceString := "" @@ -444,7 +693,7 @@ func (ci CollectionItem) RowFromCollectionItem(currency string) []string { strconv.Itoa(ci.FolderID), strconv.Itoa(ci.ID), strconv.Itoa(ci.InstanceID), - ci.BasicInformation.Artists[0].Name + " - " + ci.BasicInformation.Title, + ci.String(), format, dateAdded.Format("2006-01-02"), notes.Price.String(), @@ -456,14 +705,18 @@ func (ci CollectionItem) RowFromCollectionItem(currency string) []string { return row } +func (ci CollectionItem) String() string { + return ci.BasicInformation.Artists[0].Name + " - " + ci.BasicInformation.Title +} + func main() { patContent, err := os.ReadFile("personal_access_token") if err != nil { - panic(noPaToken) + panic(errNoPersonalAccessToken) } userAndToken := strings.Split(string(patContent), ":") if len(userAndToken) != 2 { - panic(noPaToken) + panic(errNoPersonalAccessToken) } c, err := NewClient(CURRENCY, USER_AGENT, userAndToken[0], strings.TrimSuffix(string(userAndToken[1]), "\n"), STORE) @@ -473,17 +726,25 @@ func main() { reloadCollection := false printCollection := false + printStats := false csvTemplatePath := "" csvApplyFromPath := "" skipRowsOnApplyFrom := 0 currency := "£" + mediaType := "all" + ratePerMinute := 60 + randomItem := 0 flag.BoolVar(&reloadCollection, "reload", false, "pass to force reload of collection cache.") flag.BoolVar(&printCollection, "print", false, "pass to print JSON representation of collection.") + flag.BoolVar(&printStats, "stats", false, "pass to print pricing stats from collection.") flag.StringVar(&csvTemplatePath, "template", "", "filepath to write template csv to.") flag.StringVar(&csvApplyFromPath, "apply", "", "filepath to csv to apply notes from.") flag.IntVar(&skipRowsOnApplyFrom, "skip", 0, "number of rows to skip applying from, useful if previous attempt failed.") flag.StringVar(¤cy, "currency", "£", "currency symbol to fill in.") + flag.StringVar(&mediaType, "media", "all", "media type to write records for (-template only!): all/cd/cassette/vinyl") + flag.IntVar(&ratePerMinute, "rate", 60, "number of requests to send a minute before waiting till the minute's passed.") + flag.IntVar(&randomItem, "random", 0, "pass != 0 to get n random items from the collection. filterable with -media.") flag.Parse() @@ -508,18 +769,35 @@ func main() { fmt.Println(v) } + if printStats { + err := c.Statistics(currency) + if err != nil { + panic(err) + } + } + if csvTemplatePath != "" { - if err := c.WriteCSVTemplate(csvTemplatePath, currency); err != nil { + if err := c.WriteCSVTemplate(csvTemplatePath, currency, mediaType); err != nil { panic(err) } } if csvApplyFromPath != "" { - n, err := c.ReadFromFilledCSV(csvApplyFromPath, currency, skipRowsOnApplyFrom) + n, err := c.ReadFromFilledCSV(csvApplyFromPath, currency, skipRowsOnApplyFrom, ratePerMinute) if err != nil { panic(fmt.Errorf("failed at %d: %v", n, err)) } } + if randomItem != 0 { + items, err := c.RandomItems(randomItem, mediaType) + if err != nil { + panic(err) + } + for i, item := range items { + fmt.Printf("%d: %s\n", i+1, item.String()) + } + } + // c.TestEditField() }