diff --git a/README.md b/README.md index 2e9752f..2ddf0e9 100644 --- a/README.md +++ b/README.md @@ -1,12 +1,7 @@ -# muzi +# muzi (go port) Self hosted music listening statistics -### Dependencies: -- cJSON (https://github.com/DaveGamble/cJSON) -- libzip (https://libzip.org/) -- PostgreSQL (https://www.postgresql.org/) - ### plans: - Ability to import all listening statistics and scrobbles from lastfm, spotify, apple music - daily, weekly, monthly, yearly, lifetime presets for listening reports diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..091545b --- /dev/null +++ b/go.mod @@ -0,0 +1,13 @@ +module muzi + +go 1.25.4 + +require ( + github.com/jackc/pgpassfile v1.0.0 // indirect + github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect + github.com/jackc/pgx v3.6.2+incompatible // indirect + github.com/jackc/pgx/v5 v5.7.6 // indirect + github.com/pkg/errors v0.9.1 // indirect + golang.org/x/crypto v0.45.0 // indirect + golang.org/x/text v0.31.0 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..a981f07 --- /dev/null +++ b/go.sum @@ -0,0 +1,21 @@ +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= +github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= +github.com/jackc/pgx v3.6.2+incompatible h1:2zP5OD7kiyR3xzRYMhOcXVvkDZsImVXfj+yIyTQf3/o= +github.com/jackc/pgx v3.6.2+incompatible/go.mod h1:0ZGrqGqkRlliWnWB4zKnWtjbSWbGkVEFm4TeybAXq+I= +github.com/jackc/pgx/v5 v5.7.6 h1:rWQc5FwZSPX58r1OQmkuaNicxdmExaEz5A2DO2hUuTk= +github.com/jackc/pgx/v5 v5.7.6/go.mod h1:aruU7o91Tc2q2cFp5h4uP3f6ztExVpyVv88Xl/8Vl8M= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +golang.org/x/crypto v0.45.0 h1:jMBrvKuj23MTlT0bQEOBcAE0mjg8mK9RXFhRH6nyF3Q= +golang.org/x/crypto v0.45.0/go.mod h1:XTGrrkGJve7CYK7J8PEww4aY7gM3qMCElcJQ8n8JdX4= +golang.org/x/text v0.31.0 h1:aC8ghyu4JhP8VojJ2lEHBnochRno1sgL6nEi9WGFGMM= +golang.org/x/text v0.31.0/go.mod h1:tKRAlv61yKIjGGHX/4tP1LTbc13YSec1pxVEWXzfoeM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/muzi.go b/muzi.go new file mode 100644 index 0000000..ff46128 --- /dev/null +++ b/muzi.go @@ -0,0 +1,219 @@ +package main + +import ( + "archive/zip" + "context" + "encoding/json" + "fmt" + "github.com/jackc/pgx/v5" + "io" + "os" + "path/filepath" + "strings" +) + +const ( + spotify = iota + lastfm + apple +) + +func tableExists(name string, conn *pgx.Conn) bool { + var exists bool + err := conn.QueryRow(context.Background(), "SELECT EXISTS (SELECT 1 FROM pg_tables WHERE schemaname = 'public' AND tablename = $1);", name).Scan(&exists) + if err != nil { + fmt.Fprintf(os.Stderr, "SELECT EXISTS failed: %v\n", err) + return false + } + return exists +} + +func dbExists() bool { + conn, err := pgx.Connect(context.Background(), "postgres://postgres:postgres@localhost:5432/muzi") + if err != nil { + fmt.Fprintf(os.Stderr, "Cannot connect to muzi database: %v\n", err) + return false + } + defer conn.Close(context.Background()) + return true +} + +func createDB() error { + conn, err := pgx.Connect(context.Background(), "postgres://postgres:postgres@localhost:5432") + if err != nil { + fmt.Fprintf(os.Stderr, "Cannot connect to PostgreSQL: %v\n", err) + return err + } + defer conn.Close(context.Background()) + _, err = conn.Exec(context.Background(), "CREATE DATABASE muzi") + if err != nil { + fmt.Fprintf(os.Stderr, "Cannot create muzi database: %v\n", err) + return err + } + return nil +} + +func jsonToDB(jsonFile string, platform int) { + if !dbExists() { + err := createDB() + if err != nil { + panic(err) + } + } + conn, err := pgx.Connect(context.Background(), "postgres://postgres:postgres@localhost:5432/muzi") + if err != nil { + fmt.Fprintf(os.Stderr, "Cannot connect to muzi database: %v\n", err) + panic(err) + } + defer conn.Close(context.Background()) + if !tableExists("history", conn) { + _, err = conn.Exec(context.Background(), "CREATE TABLE history ( ms_played INTEGER, timestamp TIMESTAMPTZ, song_name TEXT, artist TEXT, album_name TEXT, PRIMARY KEY (timestamp, ms_played, artist, song_name));") + } + if err != nil { + fmt.Fprintf(os.Stderr, "Cannot create history table: %v\n", err) + panic(err) + } + jsonData, err := os.ReadFile(jsonFile) + if err != nil { + fmt.Fprintf(os.Stderr, "Cannot read %s: %v\n", jsonFile, err) + panic(err) + } + if platform == spotify { + type Track struct { + Timestamp string `json:"ts"` + //Platform string `json:"platform"` + Played int `json:"ms_played"` + //Country string `json:"conn_country"` + //IP string `json:"ip_addr"` + Name string `json:"master_metadata_track_name"` + Artist string `json:"master_metadata_album_artist_name"` + Album string `json:"master_metadata_album_album_name"` + //TrackURI string `json:"spotify_track_uri"` + //Episode string `json:"episode_name"` + //Show string `json:"episode_show_name"` + //EpisodeURI string `json:"spotify_episode_uri"` + //Audiobook string `json:"audiobook_title"` + //AudiobookURI string `json:"audiobook_uri"` + //AudiobookChapterURI string `json:"audiobook_chapter_uri"` + //AudiobookChapter string `json:"audiobook_chapter_title"` + //ReasonStart string `json:"reason_start"` + //ReasonEnd string `json:"reason_end"` + //Shuffle bool `json:"shuffle"` + //Skipped bool `json:"skipped"` + //Offline bool `json:"offline"` + //OfflineTimestamp int `json:"offline_timestamp"` + //Incognito bool `json:"incognito_mode"` + } + var tracks []Track + err := json.Unmarshal(jsonData, &tracks) + if err != nil { + fmt.Fprintf(os.Stderr, "Cannot unmarshal %s: %v\n", jsonFile, err) + panic(err) + } + for _, track := range tracks { + // skip adding a song if it was only listed to for less than 20 seconds + if track.Played < 20000 { + continue + } + _, err = conn.Exec(context.Background(), "INSERT INTO history (timestamp, song_name, artist, album_name, ms_played) VALUES ($1, $2, $3, $4, $5);", track.Timestamp, track.Name, track.Artist, track.Album, track.Played) + if err != nil { + fmt.Fprintf(os.Stderr, "Couldn't add track to muzi DB (%s): %v\n", (track.Artist + " - " + track.Name), err) + } + } + } +} + +func addDirToDB(path string, platform int) { + dirs, err := os.ReadDir(path) + if err != nil { + panic(err) + } + for _, dir := range dirs { + subPath := filepath.Join(path, dir.Name(), "Spotify Extended Streaming History") + entries, err := os.ReadDir(subPath) + if err != nil { + panic(err) + } + for _, f := range entries { + jsonFileName := f.Name() + if platform == spotify { + if !strings.Contains(jsonFileName, ".json") { + continue + } + // prevents parsing spotify video data that causes duplicates + if strings.Contains(jsonFileName, "Video") { + continue + } + } + jsonFilePath := filepath.Join(subPath, jsonFileName) + jsonToDB(jsonFilePath, platform) + } + } +} + +func importSpotify() { + path := filepath.Join(".", "spotify-data", "zip") + targetBase := filepath.Join(".", "spotify-data", "extracted") + entries, err := os.ReadDir(path) + if err != nil { + panic(err) + } + for _, f := range entries { + _, err := zip.OpenReader(filepath.Join(path, f.Name())) + if err == nil { + fileName := f.Name() + fileFullPath := filepath.Join(path, fileName) + fileBaseName := fileName[:(strings.LastIndex(fileName, "."))] + targetDirFullPath := filepath.Join(targetBase, fileBaseName) + + extract(fileFullPath, targetDirFullPath) + } + } + addDirToDB(targetBase, spotify) +} + +func extract(path string, target string) { + archive, err := zip.OpenReader(path) + if err != nil { + panic(err) + } + defer archive.Close() + + zipDir := filepath.Base(path) + zipDir = zipDir[:(strings.LastIndex(zipDir, "."))] + + for _, f := range archive.File { + filePath := filepath.Join(target, f.Name) + fmt.Println("extracting:", filePath) + + if !strings.HasPrefix(filePath, filepath.Clean(target)+string(os.PathSeparator)) { + fmt.Println("Invalid file path") + return + } + if f.FileInfo().IsDir() { + fmt.Println("Creating Directory", filePath) + os.MkdirAll(filePath, os.ModePerm) + continue + } + if err := os.MkdirAll(filepath.Dir(filePath), os.ModePerm); err != nil { + panic(err) + } + fileToExtract, err := os.OpenFile(filePath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, f.Mode()) + if err != nil { + panic(err) + } + extractedFile, err := f.Open() + if err != nil { + panic(err) + } + if _, err := io.Copy(fileToExtract, extractedFile); err != nil { + panic(err) + } + fileToExtract.Close() + extractedFile.Close() + } +} + +func main() { + importSpotify() +}