diff --git a/importsongs/importsongs.go b/importsongs/importsongs.go
deleted file mode 100644
index 1f4fd04..0000000
--- a/importsongs/importsongs.go
+++ /dev/null
@@ -1,436 +0,0 @@
-package importsongs
-
-import (
- "archive/zip"
- "context"
- "encoding/json"
- "fmt"
- "io"
- "net/http"
- "os"
- "path/filepath"
- "strconv"
- "strings"
- "time"
-
- "github.com/jackc/pgx/v5"
-)
-
-const (
- spotify = iota
- lastfm
- apple
-)
-
-func TableExists(name string, conn *pgx.Conn) bool {
- var exists bool
- err := conn.QueryRow(
- context.Background(),
- `SELECT EXISTS (SELECT 1 FROM pg_tables WHERE schemaname = 'public' AND
- tablename = $1);`,
- name,
- ).
- Scan(&exists)
- if err != nil {
- fmt.Fprintf(os.Stderr, "SELECT EXISTS failed: %v\n", err)
- return false
- }
- return exists
-}
-
-func DbExists() bool {
- conn, err := pgx.Connect(
- context.Background(),
- "postgres://postgres:postgres@localhost:5432/muzi",
- )
- if err != nil {
- return false
- }
- defer conn.Close(context.Background())
- return true
-}
-
-func CreateDB() error {
- conn, err := pgx.Connect(
- context.Background(),
- "postgres://postgres:postgres@localhost:5432",
- )
- if err != nil {
- fmt.Fprintf(os.Stderr, "Cannot connect to PostgreSQL: %v\n", err)
- return err
- }
- defer conn.Close(context.Background())
- _, err = conn.Exec(context.Background(), "CREATE DATABASE muzi")
- if err != nil {
- fmt.Fprintf(os.Stderr, "Cannot create muzi database: %v\n", err)
- return err
- }
- return nil
-}
-
-func JsonToDB(jsonFile string, platform int) error {
- if !DbExists() {
- err := CreateDB()
- if err != nil {
- fmt.Fprintf(os.Stderr, "Error creating muzi database: %v\n", err)
- panic(err)
- }
- }
- conn, err := pgx.Connect(
- context.Background(),
- "postgres://postgres:postgres@localhost:5432/muzi",
- )
- if err != nil {
- fmt.Fprintf(os.Stderr, "Cannot connect to muzi database: %v\n", err)
- panic(err)
- }
- defer conn.Close(context.Background())
- if !TableExists("history", conn) {
- _, err = conn.Exec(
- context.Background(),
- `CREATE TABLE history ( ms_played INTEGER, timestamp TIMESTAMPTZ,
- song_name TEXT, artist TEXT, album_name TEXT, PRIMARY KEY (timestamp,
- ms_played, artist, song_name));`,
- )
- }
- if err != nil {
- fmt.Fprintf(os.Stderr, "Cannot create history table: %v\n", err)
- panic(err)
- }
- jsonData, err := os.ReadFile(jsonFile)
- if err != nil {
- fmt.Fprintf(os.Stderr, "Cannot read %s: %v\n", jsonFile, err)
- return err
- }
- if platform == spotify {
- type Track struct {
- Timestamp string `json:"ts"`
- Platform string `json:"-"`
- Played int `json:"ms_played"`
- Country string `json:"-"`
- IP string `json:"-"`
- Name string `json:"master_metadata_track_name"`
- Artist string `json:"master_metadata_album_artist_name"`
- Album string `json:"master_metadata_album_album_name"`
- TrackURI string `json:"-"`
- Episode string `json:"-"`
- Show string `json:"-"`
- EpisodeURI string `json:"-"`
- Audiobook string `json:"-"`
- AudiobookURI string `json:"-"`
- AudiobookChapterURI string `json:"-"`
- AudiobookChapter string `json:"-"`
- ReasonStart string `json:"-"`
- ReasonEnd string `json:"-"`
- Shuffle bool `json:"-"`
- Skipped bool `json:"-"`
- Offline bool `json:"-"`
- OfflineTimestamp int `json:"-"`
- Incognito bool `json:"-"`
- }
- var tracks []Track
- err := json.Unmarshal(jsonData, &tracks)
- if err != nil {
- fmt.Fprintf(os.Stderr, "Cannot unmarshal %s: %v\n", jsonFile, err)
- return err
- }
- for _, track := range tracks {
- // skip adding a song if it was only listed to for less than 20 seconds
- if track.Played < 20000 {
- continue
- }
- _, err = conn.Exec(
- context.Background(),
- `INSERT INTO history (timestamp, song_name, artist, album_name,
- ms_played) VALUES ($1, $2, $3, $4, $5);`,
- track.Timestamp,
- track.Name,
- track.Artist,
- track.Album,
- track.Played,
- )
- if err != nil {
- fmt.Fprintf(
- os.Stderr,
- "Couldn't add track to muzi DB (%s): %v\n",
- (track.Artist + " - " + track.Name),
- err,
- )
- }
- }
- }
- return nil
-}
-
-func AddDirToDB(path string, platform int) error {
- dirs, err := os.ReadDir(path)
- if err != nil {
- fmt.Fprintf(os.Stderr, "Error while reading path: %s: %v\n", path, err)
- return err
- }
- for _, dir := range dirs {
- subPath := filepath.Join(
- path,
- dir.Name(),
- "Spotify Extended Streaming History",
- )
- entries, err := os.ReadDir(subPath)
- if err != nil {
- fmt.Fprintf(os.Stderr, "Error while reading path: %s: %v\n", subPath, err)
- return err
- }
- for _, f := range entries {
- jsonFileName := f.Name()
- if platform == spotify {
- if !strings.Contains(jsonFileName, ".json") {
- continue
- }
- // prevents parsing spotify video data that causes duplicates
- if strings.Contains(jsonFileName, "Video") {
- continue
- }
- }
- jsonFilePath := filepath.Join(subPath, jsonFileName)
- err = JsonToDB(jsonFilePath, platform)
- if err != nil {
- fmt.Fprintf(os.Stderr,
- "Error adding json data (%s) to muzi database: %v", jsonFilePath, err)
- return err
- }
- }
- }
- return nil
-}
-
-func ImportLastFM(username string, apiKey string) error {
- if !DbExists() {
- err := CreateDB()
- if err != nil {
- fmt.Fprintf(os.Stderr, "Error creating muzi database: %v\n", err)
- panic(err)
- }
- }
- conn, err := pgx.Connect(
- context.Background(),
- "postgres://postgres:postgres@localhost:5432/muzi",
- )
- if err != nil {
- fmt.Fprintf(os.Stderr, "Cannot connect to muzi database: %v\n", err)
- panic(err)
- }
- defer conn.Close(context.Background())
- if !TableExists("history", conn) {
- _, err = conn.Exec(
- context.Background(),
- `CREATE TABLE history ( ms_played INTEGER, timestamp TIMESTAMPTZ,
- song_name TEXT, artist TEXT, album_name TEXT, PRIMARY KEY (timestamp,
- ms_played, artist, song_name));`,
- )
- }
- if err != nil {
- fmt.Fprintf(os.Stderr, "Cannot create history table: %v\n", err)
- panic(err)
- }
-
- resp, err := http.Get(
- "https://ws.audioscrobbler.com/2.0/?method=user.getrecenttracks&user=" +
- username + "&api_key=" + apiKey + "&format=json&limit=1",
- )
- if err != nil {
- fmt.Fprintf(os.Stderr, "Error getting LastFM http response: %v\n", err)
- return err
- }
- type Response struct {
- Recenttracks struct {
- Track []struct {
- Artist struct {
- Mbid string `json:"-"`
- Text string `json:"#text"`
- } `json:"artist"`
- Streamable string `json:"-"`
- Image []struct {
- Size string `json:"-"`
- Text string `json:"-"`
- } `json:"-"`
- Mbid string `json:"-"`
- Album struct {
- Mbid string `json:"-"`
- Text string `json:"#text"`
- } `json:"album"`
- Name string `json:"name"`
- Attr struct {
- Nowplaying string `json:"nowplaying"`
- } `json:"@attr,omitempty"`
- URL string `json:"-"`
- Date struct {
- Uts string `json:"uts"`
- Text string `json:"-"`
- } `json:"date"`
- } `json:"track"`
- Attr struct {
- PerPage string `json:"-"`
- TotalPages string `json:"totalPages"`
- Page string `json:"page"`
- Total string `json:"-"`
- User string `json:"-"`
- } `json:"@attr"`
- } `json:"recenttracks"`
- }
- var data Response
- json.NewDecoder(resp.Body).Decode(&data)
- totalPages, err := strconv.Atoi(data.Recenttracks.Attr.TotalPages)
- if totalPages%100 != 0 {
- totalPages = totalPages / 100
- totalPages++
- } else {
- totalPages = totalPages / 100
- }
-
- for i := 1; i <= totalPages; i++ {
- resp, err := http.Get(
- "https://ws.audioscrobbler.com/2.0/?method=user.getrecenttracks&user=" +
- username + "&api_key=" + apiKey + "&format=json&limit=100&page=" +
- strconv.Itoa(
- i,
- ),
- )
- if err != nil {
- fmt.Fprintf(os.Stderr, "Error getting LastFM http response: %v\n", err)
- return err
- }
- json.NewDecoder(resp.Body).Decode(&data)
- for j := range data.Recenttracks.Track {
- if data.Recenttracks.Track[j].Attr.Nowplaying == "true" {
- continue
- }
- unixTime, err := strconv.ParseInt(
- data.Recenttracks.Track[j].Date.Uts,
- 10,
- 64,
- )
- if err != nil {
- fmt.Fprintf(os.Stderr, "Error parsing string for int: %v\n", err)
- return err
- }
- ts := time.Unix(unixTime, 0)
- _, err = conn.Exec(
- context.Background(),
- `INSERT INTO history (timestamp, song_name, artist, album_name,
- ms_played) VALUES ($1, $2, $3, $4, $5);`,
- ts,
- data.Recenttracks.Track[j].Name,
- data.Recenttracks.Track[j].Artist.Text,
- data.Recenttracks.Track[j].Album.Text,
- 0,
- )
- if err != nil {
- fmt.Fprintf(
- os.Stderr,
- "Couldn't add track to muzi DB (%s): %v\n",
- (data.Recenttracks.Track[j].Artist.Text + " - " +
- data.Recenttracks.Track[j].Name), err)
- }
- }
- }
- return nil
-}
-
-func ImportSpotify() error {
- path := filepath.Join(".", "imports", "spotify", "zip")
- targetBase := filepath.Join(".", "imports", "spotify", "extracted")
- entries, err := os.ReadDir(path)
- if err != nil {
- fmt.Fprintf(os.Stderr, "Error reading path: %s: %v\n", path, err)
- return err
- }
- for _, f := range entries {
- _, err := zip.OpenReader(filepath.Join(path, f.Name()))
- if err != nil {
- fmt.Fprintf(os.Stderr, "Error opening zip: %s: %v\n",
- filepath.Join(path, f.Name()), err)
- continue
- }
- fileName := f.Name()
- fileFullPath := filepath.Join(path, fileName)
- fileBaseName := fileName[:(strings.LastIndex(fileName, "."))]
- targetDirFullPath := filepath.Join(targetBase, fileBaseName)
-
- err = Extract(fileFullPath, targetDirFullPath)
- if err != nil {
- fmt.Fprintf(os.Stderr, "Error extracting %s to %s: %v\n",
- fileFullPath, targetDirFullPath, err)
- return err
- }
- }
- err = AddDirToDB(targetBase, spotify)
- if err != nil {
- fmt.Fprintf(os.Stderr,
- "Error adding directory of data (%s) to muzi database: %v\n",
- targetBase, err)
- return err
- }
- return nil
-}
-
-func Extract(path string, target string) error {
- archive, err := zip.OpenReader(path)
- if err != nil {
- fmt.Fprintf(os.Stderr, "Error opening zip: %s: %v\n", path, err)
- return err
- }
- defer archive.Close()
-
- zipDir := filepath.Base(path)
- zipDir = zipDir[:(strings.LastIndex(zipDir, "."))]
-
- for _, f := range archive.File {
- filePath := filepath.Join(target, f.Name)
- fmt.Println("extracting:", filePath)
-
- if !strings.HasPrefix(
- filePath,
- filepath.Clean(target)+string(os.PathSeparator),
- ) {
- err = fmt.Errorf("Invalid file path: %s", filePath)
- fmt.Fprintf(os.Stderr, "%v\n", err)
- return err
- }
- if f.FileInfo().IsDir() {
- fmt.Println("Creating Directory", filePath)
- os.MkdirAll(filePath, os.ModePerm)
- continue
- }
- if err := os.MkdirAll(filepath.Dir(filePath), os.ModePerm); err != nil {
- fmt.Fprintf(os.Stderr, "Error making directory: %s: %v\n",
- filepath.Dir(filePath), err)
- return err
- }
- fileToExtract, err := os.OpenFile(
- filePath,
- os.O_WRONLY|os.O_CREATE|os.O_TRUNC,
- f.Mode(),
- )
- if err != nil {
- fmt.Fprintf(os.Stderr, "Error opening file: %s: %v\n", filePath, err)
- return err
- }
- extractedFile, err := f.Open()
- if err != nil {
- fmt.Fprintf(os.Stderr, "Error opening file: %s: %v\n", f.Name, err)
- return err
- }
- if _, err := io.Copy(fileToExtract, extractedFile); err != nil {
- fmt.Fprintf(
- os.Stderr,
- "Error while copying file: %s to: %s: %v\n",
- fileToExtract.Name(),
- extractedFile,
- err,
- )
- return err
- }
- fileToExtract.Close()
- extractedFile.Close()
- }
- return nil
-}
diff --git a/main.go b/main.go
index f2ab8c8..7fdf755 100644
--- a/main.go
+++ b/main.go
@@ -6,13 +6,13 @@ import (
"os"
"path/filepath"
- "muzi/importsongs"
+ "muzi/migrate"
"muzi/web"
)
func dbCheck() error {
- if !importsongs.DbExists() {
- err := importsongs.CreateDB()
+ if !migrate.DbExists() {
+ err := migrate.CreateDB()
if err != nil {
fmt.Fprintf(os.Stderr, "Error creating muzi DB: %v\n", err)
return err
@@ -70,12 +70,12 @@ func main() {
username := ""
apiKey := ""
- fmt.Printf("Importing LastFM data for %s", username)
- err = importsongs.ImportLastFM(username, apiKey)
+ fmt.Printf("Importing LastFM data for %s\n", username)
+ err = migrate.ImportLastFM(username, apiKey)
if err != nil {
return
}
- err = importsongs.ImportSpotify()
+ err = migrate.ImportSpotify()
if err != nil {
return
}
diff --git a/migrate/lastfm.go b/migrate/lastfm.go
new file mode 100644
index 0000000..90fa03b
--- /dev/null
+++ b/migrate/lastfm.go
@@ -0,0 +1,236 @@
+package migrate
+
+import (
+ "context"
+ "encoding/json"
+ "fmt"
+ "net/http"
+ "os"
+ "strconv"
+ "strings"
+ "sync"
+ "time"
+
+ "github.com/jackc/pgx/v5"
+)
+
+type LastFMTrack struct {
+ Timestamp time.Time
+ SongName string
+ Artist string
+ Album string
+}
+
+type pageResult struct {
+ pageNum int
+ tracks []LastFMTrack
+ err error
+}
+
+type Response struct {
+ Recenttracks struct {
+ Track []struct {
+ Artist struct {
+ Text string `json:"#text"`
+ } `json:"artist"`
+ Album struct {
+ Text string `json:"#text"`
+ } `json:"album"`
+ Name string `json:"name"`
+ Attr struct {
+ Nowplaying string `json:"nowplaying"`
+ } `json:"@attr,omitempty"`
+ Date struct {
+ Uts string `json:"uts"`
+ } `json:"date"`
+ } `json:"track"`
+ Attr struct {
+ TotalPages string `json:"totalPages"`
+ } `json:"@attr"`
+ } `json:"recenttracks"`
+}
+
+func ImportLastFM(username string, apiKey string) error {
+ if !DbExists() {
+ err := CreateDB()
+ if err != nil {
+ fmt.Fprintf(os.Stderr, "Error creating muzi database: %v\n", err)
+ panic(err)
+ }
+ }
+ conn, err := pgx.Connect(
+ context.Background(),
+ "postgres://postgres:postgres@localhost:5432/muzi",
+ )
+ if err != nil {
+ fmt.Fprintf(os.Stderr, "Cannot connect to muzi database: %v\n", err)
+ panic(err)
+ }
+ defer conn.Close(context.Background())
+
+ if !TableExists("history", conn) {
+ _, err = conn.Exec(
+ context.Background(),
+ `CREATE TABLE history ( ms_played INTEGER, timestamp TIMESTAMPTZ,
+ song_name TEXT, artist TEXT, album_name TEXT, PRIMARY KEY (timestamp,
+ ms_played, artist, song_name));`,
+ )
+ if err != nil {
+ fmt.Fprintf(os.Stderr, "Cannot create history table: %v\n", err)
+ panic(err)
+ }
+ }
+
+ totalImported := 0
+
+ resp, err := http.Get(
+ "https://ws.audioscrobbler.com/2.0/?method=user.getrecenttracks&user=" +
+ username + "&api_key=" + apiKey + "&format=json&limit=100",
+ )
+ if err != nil {
+ fmt.Fprintf(os.Stderr, "Error getting LastFM HTTP response: %v\n", err)
+ return err
+ }
+ var initialData Response
+ json.NewDecoder(resp.Body).Decode(&initialData)
+ totalPages, err := strconv.Atoi(initialData.Recenttracks.Attr.TotalPages)
+ resp.Body.Close()
+ if err != nil {
+ fmt.Fprintf(os.Stderr, "Error parsing total pages: %v\n", err)
+ return err
+ }
+ fmt.Printf("Total pages: %d\n", totalPages)
+
+ trackBatch := make([]LastFMTrack, 0, 1000)
+
+ pageChan := make(chan pageResult, 20)
+
+ var wg sync.WaitGroup
+ // use 10 workers
+ wg.Add(10)
+
+ for worker := range 10 {
+ go func(workerID int) {
+ defer wg.Done()
+ // distrubute 10 pages to each worker
+ for page := workerID + 1; page <= totalPages; page += 10 {
+ resp, err := http.Get(
+ "https://ws.audioscrobbler.com/2.0/?method=user.getrecenttracks&user=" +
+ username + "&api_key=" + apiKey + "&format=json&limit=100&page=" + strconv.Itoa(page),
+ )
+ if err != nil {
+ pageChan <- pageResult{pageNum: page, err: err}
+ continue
+ }
+ var data Response
+ if err := json.NewDecoder(resp.Body).Decode(&data); err != nil {
+ resp.Body.Close()
+ pageChan <- pageResult{pageNum: page, err: err}
+ continue
+ }
+ resp.Body.Close()
+
+ var pageTracks []LastFMTrack
+ for j := range data.Recenttracks.Track {
+ if data.Recenttracks.Track[j].Attr.Nowplaying == "true" {
+ continue
+ }
+ unixTime, err := strconv.ParseInt(data.Recenttracks.Track[j].Date.Uts, 10, 64)
+ if err != nil {
+ continue
+ }
+ pageTracks = append(pageTracks, LastFMTrack{
+ Timestamp: time.Unix(unixTime, 0),
+ SongName: data.Recenttracks.Track[j].Name,
+ Artist: data.Recenttracks.Track[j].Artist.Text,
+ Album: data.Recenttracks.Track[j].Album.Text,
+ })
+ }
+ pageChan <- pageResult{pageNum: page, tracks: pageTracks, err: nil}
+ }
+ }(worker)
+ }
+
+ go func() {
+ wg.Wait()
+ close(pageChan)
+ }()
+
+ batchSize := 500
+
+ for result := range pageChan {
+ if result.err != nil {
+ fmt.Fprintf(os.Stderr, "Error on page %d: %v\n", result.pageNum, result.err)
+ continue
+ }
+ trackBatch = append(trackBatch, result.tracks...)
+ for len(trackBatch) >= batchSize {
+ batch := trackBatch[:batchSize]
+ trackBatch = trackBatch[batchSize:]
+ err := insertBatch(conn, batch, &totalImported, batchSize)
+ if err != nil {
+ fmt.Fprintf(os.Stderr, "Batch insert failed: %v\n", err)
+ }
+ }
+ fmt.Printf("Processed page %d/%d\n", result.pageNum, totalPages)
+ }
+
+ if len(trackBatch) > 0 {
+ err := insertBatch(conn, trackBatch, &totalImported, batchSize)
+ if err != nil {
+ fmt.Fprintf(os.Stderr, "Final batch insert failed: %v\n", err)
+ }
+ }
+
+ fmt.Printf("%d tracks imported from LastFM for user %s\n", totalImported, username)
+ return nil
+}
+
+func insertBatch(conn *pgx.Conn, tracks []LastFMTrack, totalImported *int, batchSize int) error {
+ tx, err := conn.Begin(context.Background())
+ if err != nil {
+ return err
+ }
+
+ var batchValues []string
+ var batchArgs []any
+
+ for i, track := range tracks {
+ batchValues = append(batchValues, fmt.Sprintf(
+ "($%d, $%d, $%d, $%d, $%d)",
+ len(
+ batchArgs,
+ )+1,
+ len(batchArgs)+2,
+ len(batchArgs)+3,
+ len(batchArgs)+4,
+ len(batchArgs)+5,
+ ))
+ batchArgs = append(batchArgs, track.Timestamp, track.SongName, track.Artist, track.Album, 0)
+
+ if len(batchValues) >= batchSize || i == len(tracks)-1 {
+ result, err := tx.Exec(
+ context.Background(),
+ `INSERT INTO history (timestamp, song_name, artist, album_name, ms_played) VALUES `+
+ strings.Join(batchValues, ", ")+` ON CONFLICT DO NOTHING;`,
+ batchArgs...,
+ )
+ if err != nil {
+ tx.Rollback(context.Background())
+ return err
+ }
+ rowsAffected := result.RowsAffected()
+ if rowsAffected > 0 {
+ *totalImported += int(rowsAffected)
+ }
+ batchValues = batchValues[:0]
+ batchArgs = batchArgs[:0]
+ }
+ }
+
+ if err := tx.Commit(context.Background()); err != nil {
+ return err
+ }
+
+ return nil
+}
diff --git a/migrate/migrate.go b/migrate/migrate.go
new file mode 100644
index 0000000..63a3faf
--- /dev/null
+++ b/migrate/migrate.go
@@ -0,0 +1,55 @@
+package migrate
+
+import (
+ "context"
+ "fmt"
+ "os"
+
+ "github.com/jackc/pgx/v5"
+)
+
+func TableExists(name string, conn *pgx.Conn) bool {
+ var exists bool
+ err := conn.QueryRow(
+ context.Background(),
+ `SELECT EXISTS (SELECT 1 FROM pg_tables WHERE schemaname = 'public' AND
+ tablename = $1);`,
+ name,
+ ).
+ Scan(&exists)
+ if err != nil {
+ fmt.Fprintf(os.Stderr, "SELECT EXISTS failed: %v\n", err)
+ return false
+ }
+ return exists
+}
+
+func DbExists() bool {
+ conn, err := pgx.Connect(
+ context.Background(),
+ "postgres://postgres:postgres@localhost:5432/muzi",
+ )
+ if err != nil {
+ return false
+ }
+ defer conn.Close(context.Background())
+ return true
+}
+
+func CreateDB() error {
+ conn, err := pgx.Connect(
+ context.Background(),
+ "postgres://postgres:postgres@localhost:5432",
+ )
+ if err != nil {
+ fmt.Fprintf(os.Stderr, "Cannot connect to PostgreSQL: %v\n", err)
+ return err
+ }
+ defer conn.Close(context.Background())
+ _, err = conn.Exec(context.Background(), "CREATE DATABASE muzi")
+ if err != nil {
+ fmt.Fprintf(os.Stderr, "Cannot create muzi database: %v\n", err)
+ return err
+ }
+ return nil
+}
diff --git a/migrate/spotify.go b/migrate/spotify.go
new file mode 100644
index 0000000..41260e1
--- /dev/null
+++ b/migrate/spotify.go
@@ -0,0 +1,332 @@
+package migrate
+
+import (
+ "archive/zip"
+ "context"
+ "encoding/json"
+ "fmt"
+ "io"
+ "os"
+ "path/filepath"
+ "strings"
+
+ "github.com/jackc/pgx/v5"
+)
+
+type SpotifyTrack struct {
+ Timestamp string `json:"ts"`
+ Platform string `json:"-"`
+ Played int `json:"ms_played"`
+ Country string `json:"-"`
+ IP string `json:"-"`
+ Name string `json:"master_metadata_track_name"`
+ Artist string `json:"master_metadata_album_artist_name"`
+ Album string `json:"master_metadata_album_album_name"`
+ TrackURI string `json:"-"`
+ Episode string `json:"-"`
+ Show string `json:"-"`
+ EpisodeURI string `json:"-"`
+ Audiobook string `json:"-"`
+ AudiobookURI string `json:"-"`
+ AudiobookChapterURI string `json:"-"`
+ AudiobookChapter string `json:"-"`
+ ReasonStart string `json:"-"`
+ ReasonEnd string `json:"-"`
+ Shuffle bool `json:"-"`
+ Skipped bool `json:"-"`
+ Offline bool `json:"-"`
+ OfflineTimestamp int `json:"-"`
+ Incognito bool `json:"-"`
+}
+
+func trackKey(t SpotifyTrack) string {
+ return fmt.Sprintf("%s|%d|%s|%s", t.Timestamp, t.Played, t.Artist, t.Name)
+}
+
+func getExistingTracks(conn *pgx.Conn, tracks []SpotifyTrack) (map[string]bool, error) {
+ if len(tracks) == 0 {
+ return map[string]bool{}, nil
+ }
+
+ var conditions []string
+ var args []any
+
+ for i, t := range tracks {
+ base := i * 4
+ conditions = append(conditions,
+ fmt.Sprintf("(timestamp=$%d AND ms_played=$%d AND artist=$%d AND song_name=$%d)",
+ base+1, base+2, base+3, base+4))
+ args = append(args, t.Timestamp, t.Played, t.Artist, t.Name)
+ }
+
+ query := fmt.Sprintf(
+ "SELECT timestamp, ms_played, artist, song_name FROM history WHERE %s",
+ strings.Join(conditions, " OR "))
+
+ rows, err := conn.Query(context.Background(), query, args...)
+ if err != nil {
+ return nil, err
+ }
+ defer rows.Close()
+
+ existing := make(map[string]bool)
+ for rows.Next() {
+ var ts string
+ var played int
+ var artist, song string
+ if err := rows.Scan(&ts, &played, &artist, &song); err != nil {
+ continue
+ }
+ key := fmt.Sprintf("%s|%d|%s|%s", ts, played, artist, song)
+ existing[key] = true
+ }
+
+ return existing, nil
+}
+
+func JsonToDB(jsonFile string) error {
+ if !DbExists() {
+ err := CreateDB()
+ if err != nil {
+ fmt.Fprintf(os.Stderr, "Error creating muzi database: %v\n", err)
+ panic(err)
+ }
+ }
+ conn, err := pgx.Connect(
+ context.Background(),
+ "postgres://postgres:postgres@localhost:5432/muzi",
+ )
+ if err != nil {
+ fmt.Fprintf(os.Stderr, "Cannot connect to muzi database: %v\n", err)
+ panic(err)
+ }
+ defer conn.Close(context.Background())
+ if !TableExists("history", conn) {
+ _, err = conn.Exec(
+ context.Background(),
+ `CREATE TABLE history ( ms_played INTEGER, timestamp TIMESTAMPTZ,
+ song_name TEXT, artist TEXT, album_name TEXT, PRIMARY KEY (timestamp,
+ ms_played, artist, song_name));`,
+ )
+ }
+ if err != nil {
+ fmt.Fprintf(os.Stderr, "Cannot create history table: %v\n", err)
+ panic(err)
+ }
+ jsonData, err := os.ReadFile(jsonFile)
+ if err != nil {
+ fmt.Fprintf(os.Stderr, "Cannot read %s: %v\n", jsonFile, err)
+ return err
+ }
+ var tracks []SpotifyTrack
+ err = json.Unmarshal(jsonData, &tracks)
+ if err != nil {
+ fmt.Fprintf(os.Stderr, "Cannot unmarshal %s: %v\n", jsonFile, err)
+ return err
+ }
+
+ totalImported := 0
+ batchSize := 1000
+
+ for batchStart := 0; batchStart < len(tracks); batchStart += batchSize {
+ batchEnd := batchStart + batchSize
+ if batchEnd > len(tracks) {
+ batchEnd = len(tracks)
+ }
+
+ var validTracks []SpotifyTrack
+ for i := batchStart; i < batchEnd; i++ {
+ if tracks[i].Played >= 20000 {
+ validTracks = append(validTracks, tracks[i])
+ }
+ }
+
+ if len(validTracks) == 0 {
+ continue
+ }
+
+ existing, err := getExistingTracks(conn, validTracks)
+ if err != nil {
+ fmt.Fprintf(os.Stderr, "Error checking existing tracks: %v\n", err)
+ continue
+ }
+
+ var batchValues []string
+ var batchArgs []any
+
+ for _, t := range validTracks {
+ key := trackKey(t)
+ if existing[key] {
+ continue
+ }
+
+ batchValues = append(batchValues, fmt.Sprintf(
+ "($%d, $%d, $%d, $%d, $%d)",
+ len(batchArgs)+1,
+ len(batchArgs)+2,
+ len(batchArgs)+3,
+ len(batchArgs)+4,
+ len(batchArgs)+5,
+ ))
+ batchArgs = append(batchArgs, t.Timestamp, t.Name, t.Artist, t.Album, t.Played)
+ }
+
+ if len(batchValues) == 0 {
+ continue
+ }
+
+ _, err = conn.Exec(
+ context.Background(),
+ `INSERT INTO history (timestamp, song_name, artist, album_name, ms_played) VALUES `+
+ strings.Join(batchValues, ", ")+
+ ` ON CONFLICT DO NOTHING;`,
+ batchArgs...,
+ )
+ if err != nil {
+ fmt.Fprintf(os.Stderr, "Batch insert failed: %v\n", err)
+ } else {
+ totalImported += len(batchValues)
+ }
+ }
+
+ fmt.Printf("%d tracks imported from %s\n", totalImported, jsonFile)
+ return nil
+}
+
+func AddDirToDB(path string) error {
+ dirs, err := os.ReadDir(path)
+ if err != nil {
+ fmt.Fprintf(os.Stderr, "Error while reading path: %s: %v\n", path, err)
+ return err
+ }
+ for _, dir := range dirs {
+ subPath := filepath.Join(
+ path,
+ dir.Name(),
+ "Spotify Extended Streaming History",
+ )
+ entries, err := os.ReadDir(subPath)
+ if err != nil {
+ fmt.Fprintf(os.Stderr, "Error while reading path: %s: %v\n", subPath, err)
+ return err
+ }
+ for _, f := range entries {
+ jsonFileName := f.Name()
+ if !strings.Contains(jsonFileName, ".json") {
+ continue
+ }
+ // prevents parsing spotify video data that causes duplicates
+ if strings.Contains(jsonFileName, "Video") {
+ continue
+ }
+ jsonFilePath := filepath.Join(subPath, jsonFileName)
+ err = JsonToDB(jsonFilePath)
+ if err != nil {
+ fmt.Fprintf(os.Stderr,
+ "Error adding json data (%s) to muzi database: %v", jsonFilePath, err)
+ return err
+ }
+ }
+ }
+ return nil
+}
+
+func ImportSpotify() error {
+ path := filepath.Join(".", "imports", "spotify", "zip")
+ targetBase := filepath.Join(".", "imports", "spotify", "extracted")
+ entries, err := os.ReadDir(path)
+ if err != nil {
+ fmt.Fprintf(os.Stderr, "Error reading path: %s: %v\n", path, err)
+ return err
+ }
+ for _, f := range entries {
+ _, err := zip.OpenReader(filepath.Join(path, f.Name()))
+ if err != nil {
+ fmt.Fprintf(os.Stderr, "Error opening zip: %s: %v\n",
+ filepath.Join(path, f.Name()), err)
+ continue
+ }
+ fileName := f.Name()
+ fileFullPath := filepath.Join(path, fileName)
+ fileBaseName := fileName[:(strings.LastIndex(fileName, "."))]
+ targetDirFullPath := filepath.Join(targetBase, fileBaseName)
+
+ err = Extract(fileFullPath, targetDirFullPath)
+ if err != nil {
+ fmt.Fprintf(os.Stderr, "Error extracting %s to %s: %v\n",
+ fileFullPath, targetDirFullPath, err)
+ return err
+ }
+ }
+ err = AddDirToDB(targetBase)
+ if err != nil {
+ fmt.Fprintf(os.Stderr,
+ "Error adding directory of data (%s) to muzi database: %v\n",
+ targetBase, err)
+ return err
+ }
+ return nil
+}
+
+func Extract(path string, target string) error {
+ archive, err := zip.OpenReader(path)
+ if err != nil {
+ fmt.Fprintf(os.Stderr, "Error opening zip: %s: %v\n", path, err)
+ return err
+ }
+ defer archive.Close()
+
+ zipDir := filepath.Base(path)
+ zipDir = zipDir[:(strings.LastIndex(zipDir, "."))]
+
+ for _, f := range archive.File {
+ filePath := filepath.Join(target, f.Name)
+ fmt.Println("extracting:", filePath)
+
+ if !strings.HasPrefix(
+ filePath,
+ filepath.Clean(target)+string(os.PathSeparator),
+ ) {
+ err = fmt.Errorf("Invalid file path: %s", filePath)
+ fmt.Fprintf(os.Stderr, "%v\n", err)
+ return err
+ }
+ if f.FileInfo().IsDir() {
+ fmt.Println("Creating Directory", filePath)
+ os.MkdirAll(filePath, os.ModePerm)
+ continue
+ }
+ if err := os.MkdirAll(filepath.Dir(filePath), os.ModePerm); err != nil {
+ fmt.Fprintf(os.Stderr, "Error making directory: %s: %v\n",
+ filepath.Dir(filePath), err)
+ return err
+ }
+ fileToExtract, err := os.OpenFile(
+ filePath,
+ os.O_WRONLY|os.O_CREATE|os.O_TRUNC,
+ f.Mode(),
+ )
+ if err != nil {
+ fmt.Fprintf(os.Stderr, "Error opening file: %s: %v\n", filePath, err)
+ return err
+ }
+ extractedFile, err := f.Open()
+ if err != nil {
+ fmt.Fprintf(os.Stderr, "Error opening file: %s: %v\n", f.Name, err)
+ return err
+ }
+ if _, err := io.Copy(fileToExtract, extractedFile); err != nil {
+ fmt.Fprintf(
+ os.Stderr,
+ "Error while copying file: %s to: %s: %v\n",
+ fileToExtract.Name(),
+ extractedFile,
+ err,
+ )
+ return err
+ }
+ fileToExtract.Close()
+ extractedFile.Close()
+ }
+ return nil
+}
diff --git a/static/assets/default.png b/static/assets/default.png
new file mode 100644
index 0000000..0fd5d10
Binary files /dev/null and b/static/assets/default.png differ
diff --git a/templates/create_account.gohtml b/templates/create_account.gohtml
index 2d7c748..1277be3 100644
--- a/templates/create_account.gohtml
+++ b/templates/create_account.gohtml
@@ -1,7 +1,7 @@
-
+
muzi | Create Account
diff --git a/templates/history.gohtml b/templates/history.gohtml
index a575777..b63218d 100644
--- a/templates/history.gohtml
+++ b/templates/history.gohtml
@@ -1,7 +1,7 @@
-
+
muzi | History
diff --git a/templates/login.gohtml b/templates/login.gohtml
index 087597d..7a85c98 100644
--- a/templates/login.gohtml
+++ b/templates/login.gohtml
@@ -1,7 +1,7 @@
-
+
muzi | Login
@@ -14,6 +14,11 @@
+ {{if .ShowError}}
+
+ Invalid credentials. Please try again.
+
+ {{end}}
diff --git a/templates/profile.gohtml b/templates/profile.gohtml
index 0aa35fa..4178237 100644
--- a/templates/profile.gohtml
+++ b/templates/profile.gohtml
@@ -1,12 +1,22 @@
-
+
muzi | {{.Username}}'s Profile
- {{.Bio}}
+
+
+
+
{{.Username}}
+ {{.Bio}}
+
+
+
101238 Listens
+ 1298 Artists
+
+