Compare commits

...

2 Commits

Author SHA1 Message Date
1e8aea4dbd
add random item selector 2025-02-27 13:06:37 +00:00
9fe8a9b6fb
read/write functionality done
with price, shipping, purchase date, seller, and additional notes
working for read and write. test included for exporting to/importing
from string form pushed to discogs, which is validated before changes
are made. worked on whole library.
2024-09-16 13:10:25 +01:00
4 changed files with 545 additions and 23 deletions

2
.gitignore vendored
View File

@ -1,3 +1,5 @@
personal_access_token
test.csv
*.csv
*.ods
collection.gob

9
go.mod
View File

@ -1,5 +1,10 @@
module git.hrfee.pw/hrfee/discogs-pricer
go 1.22.4
go 1.23.0
require github.com/hrfee/go-discogs v0.0.0-20240905170225-50ee0e6a98ae // indirect
toolchain go1.23.2
require (
github.com/hrfee/go-discogs v0.0.0-20240905170225-50ee0e6a98ae // indirect
golang.org/x/exp v0.0.0-20250218142911-aa4b98e5adaa // indirect
)

2
go.sum
View File

@ -2,3 +2,5 @@ github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ=
github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/hrfee/go-discogs v0.0.0-20240905170225-50ee0e6a98ae h1:9uX6ArLOVUzryMkT3VKud4c/5vNwWEwdR6vJHHvh2tw=
github.com/hrfee/go-discogs v0.0.0-20240905170225-50ee0e6a98ae/go.mod h1:Z34PS6moBg1QdybW9LkrqciRJHKdyQ9hpxvaiH8sFic=
golang.org/x/exp v0.0.0-20250218142911-aa4b98e5adaa h1:t2QcU6V556bFjYgu4L6C+6VrCPyJZ+eyRsABUPs1mz4=
golang.org/x/exp v0.0.0-20250218142911-aa4b98e5adaa/go.mod h1:BHOTPb3L19zxehTsLoJXVaTktb06DFgmdW6Wb9s8jqk=

555
main.go
View File

@ -7,14 +7,25 @@ import (
"errors"
"flag"
"fmt"
"io"
"os"
"strconv"
"strings"
"time"
"unicode"
"math/rand"
"github.com/hrfee/go-discogs"
)
const (
CustomFieldPrice discogs.FieldID = 4
CustomFieldShipping = 5
CustomFieldPurchaseDate = 6
CustomFieldFrom = 7
)
var (
CURRENCY = "GBP"
USER_AGENT = "discogs-pricer/0.0 +https://git.hrfee.pw/hrfee/discogs-pricer"
@ -25,7 +36,7 @@ var (
NotesFieldID = 4 // Test, otherwise use 3
noPaToken = errors.New(`put "hrfee:<personal access token>" in ./personal_access_token and re-run.`)
errNoPersonalAccessToken = errors.New(`put "<username>:<personal access token>" in ./personal_access_token and re-run`)
)
type ItemHash string
@ -140,7 +151,7 @@ func (c *Client) CollectionJSON() (string, error) {
return string(v), nil
}
func (c *Client) WriteCSVTemplate(path string) error {
func (c *Client) WriteCSVTemplate(path string, currency string, mediaType string) error {
if c.CollectionCache.Order == nil || len(c.CollectionCache.Order) == 0 {
if err := c.PopulateCollectionCache(); err != nil {
return err
@ -149,26 +160,339 @@ func (c *Client) WriteCSVTemplate(path string) error {
return err
}
}
acceptTypes := []string{}
if mediaType == "cd" || mediaType == "all" {
acceptTypes = append(acceptTypes, "CD")
}
if mediaType == "cassette" || mediaType == "all" {
acceptTypes = append(acceptTypes, "Cassette")
}
if mediaType == "vinyl" || mediaType == "all" {
acceptTypes = append(acceptTypes, "Vinyl")
}
file, err := os.Create(path)
if err != nil {
return err
}
defer file.Close()
w := csv.NewWriter(file)
defer w.Flush()
w.Write(HeaderRow)
for _, ciHash := range c.CollectionCache.Order {
w.Write(c.CollectionCache.Items[ciHash].RowFromCollectionItem())
l := c.CollectionCache.Items[ciHash].RowFromCollectionItem(currency)
accept := false
for _, mt := range acceptTypes {
if l[4] == mt {
accept = true
break
}
}
if !accept {
continue
}
err := w.Write(l)
if err != nil {
err = fmt.Errorf("failed writing line \"%s\": %v", l, err)
return err
}
}
return nil
}
func (c *Client) RandomItems(n int, mediaType string) ([]CollectionItem, error) {
if c.CollectionCache.Order == nil || len(c.CollectionCache.Order) == 0 {
if err := c.PopulateCollectionCache(); err != nil {
return []CollectionItem{}, err
}
if err := c.StoreCollectionCache(); err != nil {
return []CollectionItem{}, err
}
}
acceptTypes := []string{}
if mediaType == "cd" || mediaType == "all" {
acceptTypes = append(acceptTypes, "CD")
}
if mediaType == "cassette" || mediaType == "all" {
acceptTypes = append(acceptTypes, "Cassette")
}
if mediaType == "vinyl" || mediaType == "all" {
acceptTypes = append(acceptTypes, "Vinyl")
}
validRows := make([]ItemHash, 0, len(c.CollectionCache.Order))
for _, ciHash := range c.CollectionCache.Order {
l := c.CollectionCache.Items[ciHash].RowFromCollectionItem("")
accept := false
for _, mt := range acceptTypes {
if l[4] == mt {
accept = true
break
}
}
if !accept {
continue
}
validRows = append(validRows, ciHash)
}
items := make([]CollectionItem, n)
for i := range n {
items[i] = c.CollectionCache.Items[validRows[rand.Intn(len(validRows))]]
}
return items, nil
}
func (c *Client) Statistics(currency string) error {
if c.CollectionCache.Order == nil || len(c.CollectionCache.Order) == 0 {
if err := c.PopulateCollectionCache(); err != nil {
return err
}
if err := c.StoreCollectionCache(); err != nil {
return err
}
}
extremePrices := [2][2]struct {
Hash ItemHash
Price float64
}{{{"", -1}, {"", 9999999}}, {{"", -1}, {"", 9999999}}} // Max, Min, with and without shipping
total := [2]float64{0, 0}
countWithFreeShipping := 0 // or in-person
countNonZero := 0
for _, ciHash := range c.CollectionCache.Order {
notes := c.CollectionCache.Items[ciHash].ParseNotes(currency)
itemPrice := notes.Price.Float()
shippingPrice := notes.ShippingPrice.Float()
totalPrice := itemPrice + shippingPrice
if !notes.Price.IsZero() || !notes.ShippingPrice.IsZero() {
countNonZero += 1
if notes.ShippingPrice.IsZero() {
countWithFreeShipping += 1
}
}
for i, price := range [2]float64{itemPrice, totalPrice} {
if price > extremePrices[i][0].Price {
extremePrices[i][0].Price = price
extremePrices[i][0].Hash = ciHash
}
if price < extremePrices[i][1].Price && price != 0.0 {
extremePrices[i][1].Price = price
extremePrices[i][1].Hash = ciHash
}
total[i] += price
}
}
mean := [2]float64{total[0], total[1]}
mean[0] /= float64(countNonZero)
mean[1] /= float64(countNonZero)
for i, title := range [2]string{"ITEM PRICE", "ITEM + SHIPPING PRICE"} {
fmt.Printf("----%s----\n", title)
for j, extremeName := range [2]string{"Maximum", "Minimum non-zero"} {
ci := c.CollectionCache.Items[extremePrices[i][j].Hash]
fmt.Printf("%s price @ %s%.2f: \"%s\"\n", extremeName, currency, extremePrices[i][j].Price, ci.String())
}
fmt.Printf("Mean price (of all priced items): %s%.2f\n", currency, mean[i])
fmt.Printf("Total price: %s%.2f\n", currency, total[i])
}
fmt.Printf("----GENERAL----\n")
pctWithFreeShipping := float64(100) * (float64(countWithFreeShipping) / float64(countNonZero))
fmt.Printf("%d / %d (%f%%) priced items had free shipping, or were in-person pickups\n", countWithFreeShipping, countNonZero, pctWithFreeShipping)
return nil
}
func WaitForRateLimit(reqs, perMinute int) bool {
// fmt.Println(reqs, perMinute)
if (float64(reqs) / float64(perMinute)) > 0.8 {
fmt.Printf("waiting for rate limit\n")
time.Sleep(60 * time.Second)
return true
}
return false
}
func (c *Client) ReadFromFilledCSV(path string, currency string, skipFirstNRows int, ratePerMinute int) (int, error) {
file, err := os.Open(path)
if err != nil {
return 0, err
}
defer file.Close()
r := csv.NewReader(file)
i := 0
requestsMade := 0
for err == nil {
var l []string
l, err = r.Read()
if err != nil {
break
}
if l[0] == "Folder ID" {
continue
}
if i < skipFirstNRows {
i++
fmt.Printf("skipped row %d = \"%s\"\n", i, l[cArtistName])
continue
}
pp := parsePrice(l[cPrice], currency)
pps := parsePrice(l[cShipping], currency)
pn := ParsedNotes{
Price: pp,
ShippingPrice: pps,
Seller: l[cSeller],
Additional: l[cAdditionalNotes],
}
pn.PurchaseDate, err = time.Parse("2006-01-02", l[cPurchaseDate])
if err != nil {
return i, err
}
/*folderID, _ := strconv.Atoi(l[cFolderID])
releaseID, _ := strconv.Atoi(l[cReleaseID])
instanceID, _ := strconv.Atoi(l[cInstanceID])*/
pnString := pn.ToString(currency)
// fmt.Printf("For \"%s\": %s\n", l[cArtistName], pnString)
regenerated := parseOldNotes(pnString, "2006-01-02T15:04:05-07:00", currency)
pn.RoughPanickingEqual(regenerated)
folderID, err := strconv.Atoi(l[cFolderID])
if err != nil {
panic(err)
}
releaseID, err := strconv.Atoi(l[cReleaseID])
if err != nil {
panic(err)
}
instanceID, err := strconv.Atoi(l[cInstanceID])
if err != nil {
panic(err)
}
// Notes
err = c.c.EditFieldsInstance(
c.Username,
folderID,
releaseID,
instanceID,
discogs.NotesField,
pn.Additional,
)
requestsMade++
if WaitForRateLimit(requestsMade, ratePerMinute) {
requestsMade = 0
}
if err != nil {
panic(fmt.Errorf("failed on %d-%d-%d \"%s\": %v", folderID, releaseID, instanceID, l[cArtistName], err))
}
// Price
err = c.c.EditFieldsInstance(
c.Username,
folderID,
releaseID,
instanceID,
CustomFieldPrice,
pn.Price.String(),
)
requestsMade++
if WaitForRateLimit(requestsMade, ratePerMinute) {
fmt.Printf("reset rate limit counter at %d\n", requestsMade)
requestsMade = 0
}
if err != nil {
panic(fmt.Errorf("failed on %d-%d-%d \"%s\": %v", folderID, releaseID, instanceID, l[cArtistName], err))
}
if !pn.ShippingPrice.IsZero() {
err = c.c.EditFieldsInstance(
c.Username,
folderID,
releaseID,
instanceID,
CustomFieldShipping,
pn.ShippingPrice.String(),
)
requestsMade++
if WaitForRateLimit(requestsMade, ratePerMinute) {
requestsMade = 0
}
if err != nil {
panic(fmt.Errorf("failed on %d-%d-%d \"%s\": %v", folderID, releaseID, instanceID, l[cArtistName], err))
}
}
// Purchase Date
err = c.c.EditFieldsInstance(
c.Username,
folderID,
releaseID,
instanceID,
CustomFieldPurchaseDate,
pn.PurchaseDate.Format("2006-01-02"),
)
requestsMade++
if WaitForRateLimit(requestsMade, ratePerMinute) {
requestsMade = 0
}
if err != nil {
panic(fmt.Errorf("failed on %d-%d-%d \"%s\": %v", folderID, releaseID, instanceID, l[cArtistName], err))
}
// From/Seller
err = c.c.EditFieldsInstance(
c.Username,
folderID,
releaseID,
instanceID,
CustomFieldFrom,
pn.Seller,
)
requestsMade++
if WaitForRateLimit(requestsMade, ratePerMinute) {
requestsMade = 0
}
if err != nil {
panic(fmt.Errorf("failed on %d-%d-%d \"%s\": %v", folderID, releaseID, instanceID, l[cArtistName], err))
}
fmt.Printf("logged change for %d = \"%s\"\n", i, l[cArtistName])
i++
}
if errors.Is(err, io.EOF) {
return i, nil
}
return i, err
}
func (c *Client) TestEditField() {
err := c.c.EditFieldsInstance(c.Username, 1, 10191384, 313879623, discogs.NotesField, "a new 2nd value!")
fmt.Printf("Error: %v\n", err)
}
const (
cFolderID = iota
cReleaseID
cInstanceID
cArtistName
cFormat
cDateAdded
cPrice
cShipping
cPurchaseDate
cSeller
cAdditionalNotes
)
var HeaderRow = strings.Split(
"Folder ID,Release ID,Instance ID,Artist - Name,Date Added,Price,Purchased Date,Additional Notes",
"Folder ID,Release ID,Instance ID,Artist - Name,Format,Date Added,Price,Shipping (p/ Item),Purchased Date,Seller,Additional Notes",
",",
)
@ -185,10 +509,10 @@ func ParseTime(t string) (time.Time, error) {
return time.Parse("2006-01-02T15:04:05-07:00", t)
}
func (ci CollectionItem) RawNotes() string {
func (ci CollectionItem) NotesByField(f discogs.FieldID) string {
notes := ""
for _, n := range ci.Notes {
if n.FieldID == discogs.NotesField {
if n.FieldID == f {
notes = n.Value
}
}
@ -196,53 +520,203 @@ func (ci CollectionItem) RawNotes() string {
}
type ParsedNotes struct {
Price string
PurchaseDate time.Time
Additional string
Price ParsedPrice
ShippingPrice ParsedPrice
PurchaseDate time.Time
Seller string
Additional string
}
func (ci CollectionItem) ParseNotes() ParsedNotes {
notes := ci.RawNotes()
func (p1 ParsedNotes) RoughPanickingEqual(p2 ParsedNotes) {
if p1.Price.String() != p2.Price.String() {
panic(fmt.Errorf("p1.Price != p2.Price: %v != %v", p1.Price, p2.Price))
}
if p1.ShippingPrice.String() != p2.ShippingPrice.String() {
panic(fmt.Errorf("p1.ShippingPrice != p2.ShippingPrice: %v != %v", p1.ShippingPrice, p2.ShippingPrice))
}
// Don't care about date
if p1.Seller != p2.Seller {
panic(fmt.Errorf("p1.Seller != p2.Seller: \"%v\" != \"%v\"", p1.Seller, p2.Seller))
}
if p1.Additional != p2.Additional {
panic(fmt.Errorf("p1.Additional != p2.Additional: \"%v\" != \"%v\"", p1.Additional, p2.Additional))
}
}
type ParsedPrice struct {
Currency string
Major, Minor int
}
func (pp ParsedPrice) String() string {
return fmt.Sprintf("%s%d.%02d", pp.Currency, pp.Major, pp.Minor)
}
func (pp ParsedPrice) Float() float64 {
return float64(pp.Major) + (float64(pp.Minor) / float64(100))
}
func (pp *ParsedPrice) FromFloat(p float64) {
pp.Major = int(p)
pp.Minor = int((float64(p) - float64(int(p))) * float64(100))
}
func (pp ParsedPrice) IsZero() bool { return pp.Major == 0 && pp.Minor == 0 }
func parsePrice(p string, currency string) ParsedPrice {
if p == "" {
return ParsedPrice{currency, 0, 0}
}
parts := strings.SplitN(p, ".", 2)
major := parts[0]
existingCurrency := ""
for i := 0; i < len(major); i++ {
if !unicode.IsDigit(rune(major[i])) {
existingCurrency += string(rune(major[i]))
} else {
major = major[i:]
break
}
}
iMajor, _ := strconv.Atoi(major)
iMinor := 0
if len(parts) > 1 {
parts[1] = strings.ReplaceAll(parts[1], " ", "")
var err error
iMinor, err = strconv.Atoi(parts[1])
if err != nil {
panic(err)
}
}
pp := ParsedPrice{
Currency: existingCurrency,
Major: iMajor,
Minor: iMinor,
}
if currency != "" {
pp.Currency = currency
}
return pp
}
func (pn ParsedNotes) ToString(currency string) string {
price := pn.Price.String()
if !pn.ShippingPrice.IsZero() {
price += " (+" + pn.ShippingPrice.String() + ")"
}
return fmt.Sprintf("Price: %s, Purchase Date: %s, From: %s. %s", price, pn.PurchaseDate.Format("2006-01-02"), pn.Seller, pn.Additional)
}
func (ci CollectionItem) ParseNotes(currency string) ParsedNotes {
return parseNotes(
ci.NotesByField(discogs.NotesField),
ci.NotesByField(CustomFieldPrice),
ci.NotesByField(CustomFieldShipping),
ci.NotesByField(CustomFieldPurchaseDate),
ci.NotesByField(CustomFieldFrom),
currency,
)
}
func parseNotes(notes string, price string, shipping string, purchaseDate string, from string, currency string) ParsedNotes {
pn := ParsedNotes{}
pn.Price = parsePrice(price, currency)
pn.ShippingPrice = parsePrice(shipping, currency)
var err error
pn.PurchaseDate, err = time.Parse("2006-01-02", purchaseDate)
if err != nil || purchaseDate == "" || pn.PurchaseDate.IsZero() {
if purchaseDate != "" {
fmt.Printf("date error: %v\n", err)
}
pn.PurchaseDate, _ = ParseTime(purchaseDate)
}
pn.Seller = from
return pn
}
func parseOldNotes(notes string, dateAdded string, currency string) ParsedNotes {
pn := ParsedNotes{}
dateString := ""
n, err := fmt.Sscanf(notes, "Purchase Price: %s, Purchase Date: %s.%s", &(pn.Price), &dateString, &(pn.Additional))
priceString := ""
if strings.HasPrefix(notes, "Price:") {
noPrice := strings.TrimPrefix(notes, "Price: ")
priceString = strings.Split(notes, ", Purchase Date")[0]
notes = "P" + strings.SplitN(noPrice, ", P", 2)[1]
}
// fmt.Printf("Parsing \"%s\"\n", notes)
n, err := fmt.Sscanf(notes, "Purchase Date: %s, From: %s. %s", &dateString, &(pn.Seller), &(pn.Additional))
if n <= 0 || err != nil {
pn.Additional = notes
}
end := strings.Split(notes, "From: ")
if len(end) > 1 {
splitInd := strings.Index(end[1], ". ")
if splitInd != -1 {
pn.Seller = end[1][:splitInd]
pn.Additional = end[1][splitInd+len(". "):]
}
}
priceSplit := strings.Split(priceString, "(+")
// fmt.Printf("ps \"%s\" split to %+v\n", priceString, priceSplit)
// fmt.Printf("got dateString \"%s\"\n", dateString)
pn.Price = parsePrice(priceSplit[0], currency)
pn.ShippingPrice = ParsedPrice{Currency: currency}
if len(priceSplit) > 1 {
pn.ShippingPrice = parsePrice(strings.ReplaceAll(priceSplit[1], ")", ""), currency)
}
dateString = strings.TrimSuffix(dateString, ",")
pn.PurchaseDate, err = time.Parse("2006-01-02", dateString)
if err != nil || dateString == "" || pn.PurchaseDate.IsZero() {
pn.PurchaseDate, _ = ParseTime(ci.DateAdded)
if dateString != "" {
fmt.Printf("date error: %v\n", err)
}
pn.PurchaseDate, _ = ParseTime(dateAdded)
}
return pn
}
func (ci CollectionItem) RowFromCollectionItem() []string {
func (ci CollectionItem) RowFromCollectionItem(currency string) []string {
dateAdded, err := ParseTime(ci.DateAdded)
if err != nil {
panic(err)
}
notes := ci.ParseNotes()
format := "?"
if len(ci.BasicInformation.Formats) != 0 {
format = ci.BasicInformation.Formats[0].Name
}
notes := ci.ParseNotes(currency)
row := []string{
strconv.Itoa(ci.FolderID),
strconv.Itoa(ci.ID),
strconv.Itoa(ci.InstanceID),
ci.BasicInformation.Artists[0].Name + " - " + ci.BasicInformation.Title,
ci.String(),
format,
dateAdded.Format("2006-01-02"),
notes.Price,
notes.Price.String(),
notes.ShippingPrice.String(),
notes.PurchaseDate.Format("2006-01-02"),
notes.Seller,
notes.Additional,
}
return row
}
func (ci CollectionItem) String() string {
return ci.BasicInformation.Artists[0].Name + " - " + ci.BasicInformation.Title
}
func main() {
patContent, err := os.ReadFile("personal_access_token")
if err != nil {
panic(noPaToken)
panic(errNoPersonalAccessToken)
}
userAndToken := strings.Split(string(patContent), ":")
if len(userAndToken) != 2 {
panic(noPaToken)
panic(errNoPersonalAccessToken)
}
c, err := NewClient(CURRENCY, USER_AGENT, userAndToken[0], strings.TrimSuffix(string(userAndToken[1]), "\n"), STORE)
@ -252,10 +726,25 @@ func main() {
reloadCollection := false
printCollection := false
printStats := false
csvTemplatePath := ""
csvApplyFromPath := ""
skipRowsOnApplyFrom := 0
currency := "£"
mediaType := "all"
ratePerMinute := 60
randomItem := 0
flag.BoolVar(&reloadCollection, "reload", false, "pass to force reload of collection cache.")
flag.BoolVar(&printCollection, "print", false, "pass to print JSON representation of collection.")
flag.BoolVar(&printStats, "stats", false, "pass to print pricing stats from collection.")
flag.StringVar(&csvTemplatePath, "template", "", "filepath to write template csv to.")
flag.StringVar(&csvApplyFromPath, "apply", "", "filepath to csv to apply notes from.")
flag.IntVar(&skipRowsOnApplyFrom, "skip", 0, "number of rows to skip applying from, useful if previous attempt failed.")
flag.StringVar(&currency, "currency", "£", "currency symbol to fill in.")
flag.StringVar(&mediaType, "media", "all", "media type to write records for (-template only!): all/cd/cassette/vinyl")
flag.IntVar(&ratePerMinute, "rate", 60, "number of requests to send a minute before waiting till the minute's passed.")
flag.IntVar(&randomItem, "random", 0, "pass != 0 to get n random items from the collection. filterable with -media.")
flag.Parse()
@ -280,11 +769,35 @@ func main() {
fmt.Println(v)
}
if csvTemplatePath != "" {
if err := c.WriteCSVTemplate(csvTemplatePath); err != nil {
if printStats {
err := c.Statistics(currency)
if err != nil {
panic(err)
}
}
if csvTemplatePath != "" {
if err := c.WriteCSVTemplate(csvTemplatePath, currency, mediaType); err != nil {
panic(err)
}
}
if csvApplyFromPath != "" {
n, err := c.ReadFromFilledCSV(csvApplyFromPath, currency, skipRowsOnApplyFrom, ratePerMinute)
if err != nil {
panic(fmt.Errorf("failed at %d: %v", n, err))
}
}
if randomItem != 0 {
items, err := c.RandomItems(randomItem, mediaType)
if err != nil {
panic(err)
}
for i, item := range items {
fmt.Printf("%d: %s\n", i+1, item.String())
}
}
// c.TestEditField()
}