cleaned up project structure and optimized lastfm and spotify migration

importsongs/importsongs.go

@@ -1,436 +0,0 @@
-package importsongs
-
-import (
-	"archive/zip"
-	"context"
-	"encoding/json"
-	"fmt"
-	"io"
-	"net/http"
-	"os"
-	"path/filepath"
-	"strconv"
-	"strings"
-	"time"
-
-	"github.com/jackc/pgx/v5"
-)
-
-const (
-	spotify = iota
-	lastfm
-	apple
-)
-
-func TableExists(name string, conn *pgx.Conn) bool {
-	var exists bool
-	err := conn.QueryRow(
-		context.Background(),
-		`SELECT EXISTS (SELECT 1 FROM pg_tables WHERE schemaname = 'public' AND 
-		tablename = $1);`,
-		name,
-	).
-		Scan(&exists)
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "SELECT EXISTS failed: %v\n", err)
-		return false
-	}
-	return exists
-}
-
-func DbExists() bool {
-	conn, err := pgx.Connect(
-		context.Background(),
-		"postgres://postgres:postgres@localhost:5432/muzi",
-	)
-	if err != nil {
-		return false
-	}
-	defer conn.Close(context.Background())
-	return true
-}
-
-func CreateDB() error {
-	conn, err := pgx.Connect(
-		context.Background(),
-		"postgres://postgres:postgres@localhost:5432",
-	)
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "Cannot connect to PostgreSQL: %v\n", err)
-		return err
-	}
-	defer conn.Close(context.Background())
-	_, err = conn.Exec(context.Background(), "CREATE DATABASE muzi")
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "Cannot create muzi database: %v\n", err)
-		return err
-	}
-	return nil
-}
-
-func JsonToDB(jsonFile string, platform int) error {
-	if !DbExists() {
-		err := CreateDB()
-		if err != nil {
-			fmt.Fprintf(os.Stderr, "Error creating muzi database: %v\n", err)
-			panic(err)
-		}
-	}
-	conn, err := pgx.Connect(
-		context.Background(),
-		"postgres://postgres:postgres@localhost:5432/muzi",
-	)
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "Cannot connect to muzi database: %v\n", err)
-		panic(err)
-	}
-	defer conn.Close(context.Background())
-	if !TableExists("history", conn) {
-		_, err = conn.Exec(
-			context.Background(),
-			`CREATE TABLE history ( ms_played INTEGER, timestamp TIMESTAMPTZ, 
-			song_name TEXT, artist TEXT, album_name TEXT, PRIMARY KEY (timestamp, 
-			ms_played, artist, song_name));`,
-		)
-	}
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "Cannot create history table: %v\n", err)
-		panic(err)
-	}
-	jsonData, err := os.ReadFile(jsonFile)
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "Cannot read %s: %v\n", jsonFile, err)
-		return err
-	}
-	if platform == spotify {
-		type Track struct {
-			Timestamp           string `json:"ts"`
-			Platform            string `json:"-"`
-			Played              int    `json:"ms_played"`
-			Country             string `json:"-"`
-			IP                  string `json:"-"`
-			Name                string `json:"master_metadata_track_name"`
-			Artist              string `json:"master_metadata_album_artist_name"`
-			Album               string `json:"master_metadata_album_album_name"`
-			TrackURI            string `json:"-"`
-			Episode             string `json:"-"`
-			Show                string `json:"-"`
-			EpisodeURI          string `json:"-"`
-			Audiobook           string `json:"-"`
-			AudiobookURI        string `json:"-"`
-			AudiobookChapterURI string `json:"-"`
-			AudiobookChapter    string `json:"-"`
-			ReasonStart         string `json:"-"`
-			ReasonEnd           string `json:"-"`
-			Shuffle             bool   `json:"-"`
-			Skipped             bool   `json:"-"`
-			Offline             bool   `json:"-"`
-			OfflineTimestamp    int    `json:"-"`
-			Incognito           bool   `json:"-"`
-		}
-		var tracks []Track
-		err := json.Unmarshal(jsonData, &tracks)
-		if err != nil {
-			fmt.Fprintf(os.Stderr, "Cannot unmarshal %s: %v\n", jsonFile, err)
-			return err
-		}
-		for _, track := range tracks {
-			// skip adding a song if it was only listed to for less than 20 seconds
-			if track.Played < 20000 {
-				continue
-			}
-			_, err = conn.Exec(
-				context.Background(),
-				`INSERT INTO history (timestamp, song_name, artist, album_name,
-				 ms_played) VALUES ($1, $2, $3, $4, $5);`,
-				track.Timestamp,
-				track.Name,
-				track.Artist,
-				track.Album,
-				track.Played,
-			)
-			if err != nil {
-				fmt.Fprintf(
-					os.Stderr,
-					"Couldn't add track to muzi DB (%s): %v\n",
-					(track.Artist + " - " + track.Name),
-					err,
-				)
-			}
-		}
-	}
-	return nil
-}
-
-func AddDirToDB(path string, platform int) error {
-	dirs, err := os.ReadDir(path)
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "Error while reading path: %s: %v\n", path, err)
-		return err
-	}
-	for _, dir := range dirs {
-		subPath := filepath.Join(
-			path,
-			dir.Name(),
-			"Spotify Extended Streaming History",
-		)
-		entries, err := os.ReadDir(subPath)
-		if err != nil {
-			fmt.Fprintf(os.Stderr, "Error while reading path: %s: %v\n", subPath, err)
-			return err
-		}
-		for _, f := range entries {
-			jsonFileName := f.Name()
-			if platform == spotify {
-				if !strings.Contains(jsonFileName, ".json") {
-					continue
-				}
-				// prevents parsing spotify video data that causes duplicates
-				if strings.Contains(jsonFileName, "Video") {
-					continue
-				}
-			}
-			jsonFilePath := filepath.Join(subPath, jsonFileName)
-			err = JsonToDB(jsonFilePath, platform)
-			if err != nil {
-				fmt.Fprintf(os.Stderr,
-					"Error adding json data (%s) to muzi database: %v", jsonFilePath, err)
-				return err
-			}
-		}
-	}
-	return nil
-}
-
-func ImportLastFM(username string, apiKey string) error {
-	if !DbExists() {
-		err := CreateDB()
-		if err != nil {
-			fmt.Fprintf(os.Stderr, "Error creating muzi database: %v\n", err)
-			panic(err)
-		}
-	}
-	conn, err := pgx.Connect(
-		context.Background(),
-		"postgres://postgres:postgres@localhost:5432/muzi",
-	)
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "Cannot connect to muzi database: %v\n", err)
-		panic(err)
-	}
-	defer conn.Close(context.Background())
-	if !TableExists("history", conn) {
-		_, err = conn.Exec(
-			context.Background(),
-			`CREATE TABLE history ( ms_played INTEGER, timestamp TIMESTAMPTZ, 
-			song_name TEXT, artist TEXT, album_name TEXT, PRIMARY KEY (timestamp, 
-			ms_played, artist, song_name));`,
-		)
-	}
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "Cannot create history table: %v\n", err)
-		panic(err)
-	}
-
-	resp, err := http.Get(
-		"https://ws.audioscrobbler.com/2.0/?method=user.getrecenttracks&user=" +
-			username + "&api_key=" + apiKey + "&format=json&limit=1",
-	)
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "Error getting LastFM http response: %v\n", err)
-		return err
-	}
-	type Response struct {
-		Recenttracks struct {
-			Track []struct {
-				Artist struct {
-					Mbid string `json:"-"`
-					Text string `json:"#text"`
-				} `json:"artist"`
-				Streamable string `json:"-"`
-				Image      []struct {
-					Size string `json:"-"`
-					Text string `json:"-"`
-				} `json:"-"`
-				Mbid  string `json:"-"`
-				Album struct {
-					Mbid string `json:"-"`
-					Text string `json:"#text"`
-				} `json:"album"`
-				Name string `json:"name"`
-				Attr struct {
-					Nowplaying string `json:"nowplaying"`
-				} `json:"@attr,omitempty"`
-				URL  string `json:"-"`
-				Date struct {
-					Uts  string `json:"uts"`
-					Text string `json:"-"`
-				} `json:"date"`
-			} `json:"track"`
-			Attr struct {
-				PerPage    string `json:"-"`
-				TotalPages string `json:"totalPages"`
-				Page       string `json:"page"`
-				Total      string `json:"-"`
-				User       string `json:"-"`
-			} `json:"@attr"`
-		} `json:"recenttracks"`
-	}
-	var data Response
-	json.NewDecoder(resp.Body).Decode(&data)
-	totalPages, err := strconv.Atoi(data.Recenttracks.Attr.TotalPages)
-	if totalPages%100 != 0 {
-		totalPages = totalPages / 100
-		totalPages++
-	} else {
-		totalPages = totalPages / 100
-	}
-
-	for i := 1; i <= totalPages; i++ {
-		resp, err := http.Get(
-			"https://ws.audioscrobbler.com/2.0/?method=user.getrecenttracks&user=" +
-				username + "&api_key=" + apiKey + "&format=json&limit=100&page=" +
-				strconv.Itoa(
-					i,
-				),
-		)
-		if err != nil {
-			fmt.Fprintf(os.Stderr, "Error getting LastFM http response: %v\n", err)
-			return err
-		}
-		json.NewDecoder(resp.Body).Decode(&data)
-		for j := range data.Recenttracks.Track {
-			if data.Recenttracks.Track[j].Attr.Nowplaying == "true" {
-				continue
-			}
-			unixTime, err := strconv.ParseInt(
-				data.Recenttracks.Track[j].Date.Uts,
-				10,
-				64,
-			)
-			if err != nil {
-				fmt.Fprintf(os.Stderr, "Error parsing string for int: %v\n", err)
-				return err
-			}
-			ts := time.Unix(unixTime, 0)
-			_, err = conn.Exec(
-				context.Background(),
-				`INSERT INTO history (timestamp, song_name, artist, album_name, 
-				ms_played) VALUES ($1, $2, $3, $4, $5);`,
-				ts,
-				data.Recenttracks.Track[j].Name,
-				data.Recenttracks.Track[j].Artist.Text,
-				data.Recenttracks.Track[j].Album.Text,
-				0,
-			)
-			if err != nil {
-				fmt.Fprintf(
-					os.Stderr,
-					"Couldn't add track to muzi DB (%s): %v\n",
-					(data.Recenttracks.Track[j].Artist.Text + " - " +
-						data.Recenttracks.Track[j].Name), err)
-			}
-		}
-	}
-	return nil
-}
-
-func ImportSpotify() error {
-	path := filepath.Join(".", "imports", "spotify", "zip")
-	targetBase := filepath.Join(".", "imports", "spotify", "extracted")
-	entries, err := os.ReadDir(path)
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "Error reading path: %s: %v\n", path, err)
-		return err
-	}
-	for _, f := range entries {
-		_, err := zip.OpenReader(filepath.Join(path, f.Name()))
-		if err != nil {
-			fmt.Fprintf(os.Stderr, "Error opening zip: %s: %v\n",
-				filepath.Join(path, f.Name()), err)
-			continue
-		}
-		fileName := f.Name()
-		fileFullPath := filepath.Join(path, fileName)
-		fileBaseName := fileName[:(strings.LastIndex(fileName, "."))]
-		targetDirFullPath := filepath.Join(targetBase, fileBaseName)
-
-		err = Extract(fileFullPath, targetDirFullPath)
-		if err != nil {
-			fmt.Fprintf(os.Stderr, "Error extracting %s to %s: %v\n",
-				fileFullPath, targetDirFullPath, err)
-			return err
-		}
-	}
-	err = AddDirToDB(targetBase, spotify)
-	if err != nil {
-		fmt.Fprintf(os.Stderr,
-			"Error adding directory of data (%s) to muzi database: %v\n",
-			targetBase, err)
-		return err
-	}
-	return nil
-}
-
-func Extract(path string, target string) error {
-	archive, err := zip.OpenReader(path)
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "Error opening zip: %s: %v\n", path, err)
-		return err
-	}
-	defer archive.Close()
-
-	zipDir := filepath.Base(path)
-	zipDir = zipDir[:(strings.LastIndex(zipDir, "."))]
-
-	for _, f := range archive.File {
-		filePath := filepath.Join(target, f.Name)
-		fmt.Println("extracting:", filePath)
-
-		if !strings.HasPrefix(
-			filePath,
-			filepath.Clean(target)+string(os.PathSeparator),
-		) {
-			err = fmt.Errorf("Invalid file path: %s", filePath)
-			fmt.Fprintf(os.Stderr, "%v\n", err)
-			return err
-		}
-		if f.FileInfo().IsDir() {
-			fmt.Println("Creating Directory", filePath)
-			os.MkdirAll(filePath, os.ModePerm)
-			continue
-		}
-		if err := os.MkdirAll(filepath.Dir(filePath), os.ModePerm); err != nil {
-			fmt.Fprintf(os.Stderr, "Error making directory: %s: %v\n",
-				filepath.Dir(filePath), err)
-			return err
-		}
-		fileToExtract, err := os.OpenFile(
-			filePath,
-			os.O_WRONLY|os.O_CREATE|os.O_TRUNC,
-			f.Mode(),
-		)
-		if err != nil {
-			fmt.Fprintf(os.Stderr, "Error opening file: %s: %v\n", filePath, err)
-			return err
-		}
-		extractedFile, err := f.Open()
-		if err != nil {
-			fmt.Fprintf(os.Stderr, "Error opening file: %s: %v\n", f.Name, err)
-			return err
-		}
-		if _, err := io.Copy(fileToExtract, extractedFile); err != nil {
-			fmt.Fprintf(
-				os.Stderr,
-				"Error while copying file: %s to: %s: %v\n",
-				fileToExtract.Name(),
-				extractedFile,
-				err,
-			)
-			return err
-		}
-		fileToExtract.Close()
-		extractedFile.Close()
-	}
-	return nil
-}

main.go

@@ -6,13 +6,13 @@ import (
 	"os"
 	"path/filepath"
 
-	"muzi/importsongs"
+	"muzi/migrate"
 	"muzi/web"
 )
 
 func dbCheck() error {
-	if !importsongs.DbExists() {
+	if !migrate.DbExists() {
-		err := importsongs.CreateDB()
+		err := migrate.CreateDB()
 		if err != nil {
 			fmt.Fprintf(os.Stderr, "Error creating muzi DB: %v\n", err)
 			return err
@@ -70,12 +70,12 @@ func main() {
 
 	username := ""
 	apiKey := ""
-	fmt.Printf("Importing LastFM data for %s", username)
+	fmt.Printf("Importing LastFM data for %s\n", username)
-	err = importsongs.ImportLastFM(username, apiKey)
+	err = migrate.ImportLastFM(username, apiKey)
 	if err != nil {
 		return
 	}
-	err = importsongs.ImportSpotify()
+	err = migrate.ImportSpotify()
 	if err != nil {
 		return
 	}

migrate/lastfm.go

@@ -0,0 +1,236 @@
+package migrate
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"os"
+	"strconv"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/jackc/pgx/v5"
+)
+
+type LastFMTrack struct {
+	Timestamp time.Time
+	SongName  string
+	Artist    string
+	Album     string
+}
+
+type pageResult struct {
+	pageNum int
+	tracks  []LastFMTrack
+	err     error
+}
+
+type Response struct {
+	Recenttracks struct {
+		Track []struct {
+			Artist struct {
+				Text string `json:"#text"`
+			} `json:"artist"`
+			Album struct {
+				Text string `json:"#text"`
+			} `json:"album"`
+			Name string `json:"name"`
+			Attr struct {
+				Nowplaying string `json:"nowplaying"`
+			} `json:"@attr,omitempty"`
+			Date struct {
+				Uts string `json:"uts"`
+			} `json:"date"`
+		} `json:"track"`
+		Attr struct {
+			TotalPages string `json:"totalPages"`
+		} `json:"@attr"`
+	} `json:"recenttracks"`
+}
+
+func ImportLastFM(username string, apiKey string) error {
+	if !DbExists() {
+		err := CreateDB()
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "Error creating muzi database: %v\n", err)
+			panic(err)
+		}
+	}
+	conn, err := pgx.Connect(
+		context.Background(),
+		"postgres://postgres:postgres@localhost:5432/muzi",
+	)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Cannot connect to muzi database: %v\n", err)
+		panic(err)
+	}
+	defer conn.Close(context.Background())
+
+	if !TableExists("history", conn) {
+		_, err = conn.Exec(
+			context.Background(),
+			`CREATE TABLE history ( ms_played INTEGER, timestamp TIMESTAMPTZ, 
+				song_name TEXT, artist TEXT, album_name TEXT, PRIMARY KEY (timestamp, 
+				ms_played, artist, song_name));`,
+		)
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "Cannot create history table: %v\n", err)
+			panic(err)
+		}
+	}
+
+	totalImported := 0
+
+	resp, err := http.Get(
+		"https://ws.audioscrobbler.com/2.0/?method=user.getrecenttracks&user=" +
+			username + "&api_key=" + apiKey + "&format=json&limit=100",
+	)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Error getting LastFM HTTP response: %v\n", err)
+		return err
+	}
+	var initialData Response
+	json.NewDecoder(resp.Body).Decode(&initialData)
+	totalPages, err := strconv.Atoi(initialData.Recenttracks.Attr.TotalPages)
+	resp.Body.Close()
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Error parsing total pages: %v\n", err)
+		return err
+	}
+	fmt.Printf("Total pages: %d\n", totalPages)
+
+	trackBatch := make([]LastFMTrack, 0, 1000)
+
+	pageChan := make(chan pageResult, 20)
+
+	var wg sync.WaitGroup
+	// use 10 workers
+	wg.Add(10)
+
+	for worker := range 10 {
+		go func(workerID int) {
+			defer wg.Done()
+			// distrubute 10 pages to each worker
+			for page := workerID + 1; page <= totalPages; page += 10 {
+				resp, err := http.Get(
+					"https://ws.audioscrobbler.com/2.0/?method=user.getrecenttracks&user=" +
+						username + "&api_key=" + apiKey + "&format=json&limit=100&page=" + strconv.Itoa(page),
+				)
+				if err != nil {
+					pageChan <- pageResult{pageNum: page, err: err}
+					continue
+				}
+				var data Response
+				if err := json.NewDecoder(resp.Body).Decode(&data); err != nil {
+					resp.Body.Close()
+					pageChan <- pageResult{pageNum: page, err: err}
+					continue
+				}
+				resp.Body.Close()
+
+				var pageTracks []LastFMTrack
+				for j := range data.Recenttracks.Track {
+					if data.Recenttracks.Track[j].Attr.Nowplaying == "true" {
+						continue
+					}
+					unixTime, err := strconv.ParseInt(data.Recenttracks.Track[j].Date.Uts, 10, 64)
+					if err != nil {
+						continue
+					}
+					pageTracks = append(pageTracks, LastFMTrack{
+						Timestamp: time.Unix(unixTime, 0),
+						SongName:  data.Recenttracks.Track[j].Name,
+						Artist:    data.Recenttracks.Track[j].Artist.Text,
+						Album:     data.Recenttracks.Track[j].Album.Text,
+					})
+				}
+				pageChan <- pageResult{pageNum: page, tracks: pageTracks, err: nil}
+			}
+		}(worker)
+	}
+
+	go func() {
+		wg.Wait()
+		close(pageChan)
+	}()
+
+	batchSize := 500
+
+	for result := range pageChan {
+		if result.err != nil {
+			fmt.Fprintf(os.Stderr, "Error on page %d: %v\n", result.pageNum, result.err)
+			continue
+		}
+		trackBatch = append(trackBatch, result.tracks...)
+		for len(trackBatch) >= batchSize {
+			batch := trackBatch[:batchSize]
+			trackBatch = trackBatch[batchSize:]
+			err := insertBatch(conn, batch, &totalImported, batchSize)
+			if err != nil {
+				fmt.Fprintf(os.Stderr, "Batch insert failed: %v\n", err)
+			}
+		}
+		fmt.Printf("Processed page %d/%d\n", result.pageNum, totalPages)
+	}
+
+	if len(trackBatch) > 0 {
+		err := insertBatch(conn, trackBatch, &totalImported, batchSize)
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "Final batch insert failed: %v\n", err)
+		}
+	}
+
+	fmt.Printf("%d tracks imported from LastFM for user %s\n", totalImported, username)
+	return nil
+}
+
+func insertBatch(conn *pgx.Conn, tracks []LastFMTrack, totalImported *int, batchSize int) error {
+	tx, err := conn.Begin(context.Background())
+	if err != nil {
+		return err
+	}
+
+	var batchValues []string
+	var batchArgs []any
+
+	for i, track := range tracks {
+		batchValues = append(batchValues, fmt.Sprintf(
+			"($%d, $%d, $%d, $%d, $%d)",
+			len(
+				batchArgs,
+			)+1,
+			len(batchArgs)+2,
+			len(batchArgs)+3,
+			len(batchArgs)+4,
+			len(batchArgs)+5,
+		))
+		batchArgs = append(batchArgs, track.Timestamp, track.SongName, track.Artist, track.Album, 0)
+
+		if len(batchValues) >= batchSize || i == len(tracks)-1 {
+			result, err := tx.Exec(
+				context.Background(),
+				`INSERT INTO history (timestamp, song_name, artist, album_name, ms_played) VALUES `+
+					strings.Join(batchValues, ", ")+` ON CONFLICT DO NOTHING;`,
+				batchArgs...,
+			)
+			if err != nil {
+				tx.Rollback(context.Background())
+				return err
+			}
+			rowsAffected := result.RowsAffected()
+			if rowsAffected > 0 {
+				*totalImported += int(rowsAffected)
+			}
+			batchValues = batchValues[:0]
+			batchArgs = batchArgs[:0]
+		}
+	}
+
+	if err := tx.Commit(context.Background()); err != nil {
+		return err
+	}
+
+	return nil
+}

migrate/migrate.go

@@ -0,0 +1,55 @@
+package migrate
+
+import (
+	"context"
+	"fmt"
+	"os"
+
+	"github.com/jackc/pgx/v5"
+)
+
+func TableExists(name string, conn *pgx.Conn) bool {
+	var exists bool
+	err := conn.QueryRow(
+		context.Background(),
+		`SELECT EXISTS (SELECT 1 FROM pg_tables WHERE schemaname = 'public' AND 
+		tablename = $1);`,
+		name,
+	).
+		Scan(&exists)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "SELECT EXISTS failed: %v\n", err)
+		return false
+	}
+	return exists
+}
+
+func DbExists() bool {
+	conn, err := pgx.Connect(
+		context.Background(),
+		"postgres://postgres:postgres@localhost:5432/muzi",
+	)
+	if err != nil {
+		return false
+	}
+	defer conn.Close(context.Background())
+	return true
+}
+
+func CreateDB() error {
+	conn, err := pgx.Connect(
+		context.Background(),
+		"postgres://postgres:postgres@localhost:5432",
+	)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Cannot connect to PostgreSQL: %v\n", err)
+		return err
+	}
+	defer conn.Close(context.Background())
+	_, err = conn.Exec(context.Background(), "CREATE DATABASE muzi")
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Cannot create muzi database: %v\n", err)
+		return err
+	}
+	return nil
+}

migrate/spotify.go

@@ -0,0 +1,332 @@
+package migrate
+
+import (
+	"archive/zip"
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/jackc/pgx/v5"
+)
+
+type SpotifyTrack struct {
+	Timestamp           string `json:"ts"`
+	Platform            string `json:"-"`
+	Played              int    `json:"ms_played"`
+	Country             string `json:"-"`
+	IP                  string `json:"-"`
+	Name                string `json:"master_metadata_track_name"`
+	Artist              string `json:"master_metadata_album_artist_name"`
+	Album               string `json:"master_metadata_album_album_name"`
+	TrackURI            string `json:"-"`
+	Episode             string `json:"-"`
+	Show                string `json:"-"`
+	EpisodeURI          string `json:"-"`
+	Audiobook           string `json:"-"`
+	AudiobookURI        string `json:"-"`
+	AudiobookChapterURI string `json:"-"`
+	AudiobookChapter    string `json:"-"`
+	ReasonStart         string `json:"-"`
+	ReasonEnd           string `json:"-"`
+	Shuffle             bool   `json:"-"`
+	Skipped             bool   `json:"-"`
+	Offline             bool   `json:"-"`
+	OfflineTimestamp    int    `json:"-"`
+	Incognito           bool   `json:"-"`
+}
+
+func trackKey(t SpotifyTrack) string {
+	return fmt.Sprintf("%s|%d|%s|%s", t.Timestamp, t.Played, t.Artist, t.Name)
+}
+
+func getExistingTracks(conn *pgx.Conn, tracks []SpotifyTrack) (map[string]bool, error) {
+	if len(tracks) == 0 {
+		return map[string]bool{}, nil
+	}
+
+	var conditions []string
+	var args []any
+
+	for i, t := range tracks {
+		base := i * 4
+		conditions = append(conditions,
+			fmt.Sprintf("(timestamp=$%d AND ms_played=$%d AND artist=$%d AND song_name=$%d)",
+				base+1, base+2, base+3, base+4))
+		args = append(args, t.Timestamp, t.Played, t.Artist, t.Name)
+	}
+
+	query := fmt.Sprintf(
+		"SELECT timestamp, ms_played, artist, song_name FROM history WHERE %s",
+		strings.Join(conditions, " OR "))
+
+	rows, err := conn.Query(context.Background(), query, args...)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+
+	existing := make(map[string]bool)
+	for rows.Next() {
+		var ts string
+		var played int
+		var artist, song string
+		if err := rows.Scan(&ts, &played, &artist, &song); err != nil {
+			continue
+		}
+		key := fmt.Sprintf("%s|%d|%s|%s", ts, played, artist, song)
+		existing[key] = true
+	}
+
+	return existing, nil
+}
+
+func JsonToDB(jsonFile string) error {
+	if !DbExists() {
+		err := CreateDB()
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "Error creating muzi database: %v\n", err)
+			panic(err)
+		}
+	}
+	conn, err := pgx.Connect(
+		context.Background(),
+		"postgres://postgres:postgres@localhost:5432/muzi",
+	)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Cannot connect to muzi database: %v\n", err)
+		panic(err)
+	}
+	defer conn.Close(context.Background())
+	if !TableExists("history", conn) {
+		_, err = conn.Exec(
+			context.Background(),
+			`CREATE TABLE history ( ms_played INTEGER, timestamp TIMESTAMPTZ, 
+			song_name TEXT, artist TEXT, album_name TEXT, PRIMARY KEY (timestamp, 
+			ms_played, artist, song_name));`,
+		)
+	}
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Cannot create history table: %v\n", err)
+		panic(err)
+	}
+	jsonData, err := os.ReadFile(jsonFile)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Cannot read %s: %v\n", jsonFile, err)
+		return err
+	}
+	var tracks []SpotifyTrack
+	err = json.Unmarshal(jsonData, &tracks)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Cannot unmarshal %s: %v\n", jsonFile, err)
+		return err
+	}
+
+	totalImported := 0
+	batchSize := 1000
+
+	for batchStart := 0; batchStart < len(tracks); batchStart += batchSize {
+		batchEnd := batchStart + batchSize
+		if batchEnd > len(tracks) {
+			batchEnd = len(tracks)
+		}
+
+		var validTracks []SpotifyTrack
+		for i := batchStart; i < batchEnd; i++ {
+			if tracks[i].Played >= 20000 {
+				validTracks = append(validTracks, tracks[i])
+			}
+		}
+
+		if len(validTracks) == 0 {
+			continue
+		}
+
+		existing, err := getExistingTracks(conn, validTracks)
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "Error checking existing tracks: %v\n", err)
+			continue
+		}
+
+		var batchValues []string
+		var batchArgs []any
+
+		for _, t := range validTracks {
+			key := trackKey(t)
+			if existing[key] {
+				continue
+			}
+
+			batchValues = append(batchValues, fmt.Sprintf(
+				"($%d, $%d, $%d, $%d, $%d)",
+				len(batchArgs)+1,
+				len(batchArgs)+2,
+				len(batchArgs)+3,
+				len(batchArgs)+4,
+				len(batchArgs)+5,
+			))
+			batchArgs = append(batchArgs, t.Timestamp, t.Name, t.Artist, t.Album, t.Played)
+		}
+
+		if len(batchValues) == 0 {
+			continue
+		}
+
+		_, err = conn.Exec(
+			context.Background(),
+			`INSERT INTO history (timestamp, song_name, artist, album_name, ms_played) VALUES `+
+				strings.Join(batchValues, ", ")+
+				` ON CONFLICT DO NOTHING;`,
+			batchArgs...,
+		)
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "Batch insert failed: %v\n", err)
+		} else {
+			totalImported += len(batchValues)
+		}
+	}
+
+	fmt.Printf("%d tracks imported from %s\n", totalImported, jsonFile)
+	return nil
+}
+
+func AddDirToDB(path string) error {
+	dirs, err := os.ReadDir(path)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Error while reading path: %s: %v\n", path, err)
+		return err
+	}
+	for _, dir := range dirs {
+		subPath := filepath.Join(
+			path,
+			dir.Name(),
+			"Spotify Extended Streaming History",
+		)
+		entries, err := os.ReadDir(subPath)
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "Error while reading path: %s: %v\n", subPath, err)
+			return err
+		}
+		for _, f := range entries {
+			jsonFileName := f.Name()
+			if !strings.Contains(jsonFileName, ".json") {
+				continue
+			}
+			// prevents parsing spotify video data that causes duplicates
+			if strings.Contains(jsonFileName, "Video") {
+				continue
+			}
+			jsonFilePath := filepath.Join(subPath, jsonFileName)
+			err = JsonToDB(jsonFilePath)
+			if err != nil {
+				fmt.Fprintf(os.Stderr,
+					"Error adding json data (%s) to muzi database: %v", jsonFilePath, err)
+				return err
+			}
+		}
+	}
+	return nil
+}
+
+func ImportSpotify() error {
+	path := filepath.Join(".", "imports", "spotify", "zip")
+	targetBase := filepath.Join(".", "imports", "spotify", "extracted")
+	entries, err := os.ReadDir(path)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Error reading path: %s: %v\n", path, err)
+		return err
+	}
+	for _, f := range entries {
+		_, err := zip.OpenReader(filepath.Join(path, f.Name()))
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "Error opening zip: %s: %v\n",
+				filepath.Join(path, f.Name()), err)
+			continue
+		}
+		fileName := f.Name()
+		fileFullPath := filepath.Join(path, fileName)
+		fileBaseName := fileName[:(strings.LastIndex(fileName, "."))]
+		targetDirFullPath := filepath.Join(targetBase, fileBaseName)
+
+		err = Extract(fileFullPath, targetDirFullPath)
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "Error extracting %s to %s: %v\n",
+				fileFullPath, targetDirFullPath, err)
+			return err
+		}
+	}
+	err = AddDirToDB(targetBase)
+	if err != nil {
+		fmt.Fprintf(os.Stderr,
+			"Error adding directory of data (%s) to muzi database: %v\n",
+			targetBase, err)
+		return err
+	}
+	return nil
+}
+
+func Extract(path string, target string) error {
+	archive, err := zip.OpenReader(path)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Error opening zip: %s: %v\n", path, err)
+		return err
+	}
+	defer archive.Close()
+
+	zipDir := filepath.Base(path)
+	zipDir = zipDir[:(strings.LastIndex(zipDir, "."))]
+
+	for _, f := range archive.File {
+		filePath := filepath.Join(target, f.Name)
+		fmt.Println("extracting:", filePath)
+
+		if !strings.HasPrefix(
+			filePath,
+			filepath.Clean(target)+string(os.PathSeparator),
+		) {
+			err = fmt.Errorf("Invalid file path: %s", filePath)
+			fmt.Fprintf(os.Stderr, "%v\n", err)
+			return err
+		}
+		if f.FileInfo().IsDir() {
+			fmt.Println("Creating Directory", filePath)
+			os.MkdirAll(filePath, os.ModePerm)
+			continue
+		}
+		if err := os.MkdirAll(filepath.Dir(filePath), os.ModePerm); err != nil {
+			fmt.Fprintf(os.Stderr, "Error making directory: %s: %v\n",
+				filepath.Dir(filePath), err)
+			return err
+		}
+		fileToExtract, err := os.OpenFile(
+			filePath,
+			os.O_WRONLY|os.O_CREATE|os.O_TRUNC,
+			f.Mode(),
+		)
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "Error opening file: %s: %v\n", filePath, err)
+			return err
+		}
+		extractedFile, err := f.Open()
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "Error opening file: %s: %v\n", f.Name, err)
+			return err
+		}
+		if _, err := io.Copy(fileToExtract, extractedFile); err != nil {
+			fmt.Fprintf(
+				os.Stderr,
+				"Error while copying file: %s to: %s: %v\n",
+				fileToExtract.Name(),
+				extractedFile,
+				err,
+			)
+			return err
+		}
+		fileToExtract.Close()
+		extractedFile.Close()
+	}
+	return nil
+}

templates/create_account.gohtml

@@ -1,7 +1,7 @@
 <!doctype html>
 <html>
   <head>
-    <link rel="stylesheet" href="static/style.css" type="text/css">
+    <link rel="stylesheet" href="/files/style.css" type="text/css">
     <title>
       muzi | Create Account
     </title>

templates/history.gohtml

@@ -1,7 +1,7 @@
 <!doctype html>
 <html>
   <head>
-    <link rel="stylesheet" href="static/style.css" type="text/css">
+    <link rel="stylesheet" href="/files/style.css" type="text/css">
     <title>
       muzi | History
     </title>

templates/login.gohtml

@@ -1,7 +1,7 @@
 <!doctype html>
 <html>
   <head>
-    <link rel="stylesheet" href="static/style.css" type="text/css">
+    <link rel="stylesheet" href="/files/style.css" type="text/css">
     <title>
       muzi | Login
     </title>
@@ -14,6 +14,11 @@
         <label for="pass">Password:</label>
         <input type="text" id="pass" name="pass"> <br> <br>
         <input type="submit" value="Login">
+        {{if .ShowError}}
+          <div class="login-error">
+            Invalid credentials. Please try again.
+          </div>
+        {{end}}
       </form>
     </div>
   </body>

templates/profile.gohtml

@@ -1,12 +1,22 @@
 <!doctype html>
 <html>
   <head>
-    <link rel="stylesheet" href="static/style.css" type="text/css">
+    <link rel="stylesheet" href="/files/style.css" type="text/css">
     <title>
       muzi | {{.Username}}'s Profile
     </title>
   </head>
   <body>
-    {{.Bio}}
+    <div class="profile-top">
+      <img src="{{.Pfp}}" alt="{{.Username}}'s avatar">
+      <div class="username-bio">
+        <h1>{{.Username}}</h1>
+        <h2>{{.Bio}}</h2>
+      </div>
+      <div class="user-stats-top">
+        <h3>101238 Listens</h3>
+        <h3>1298 Artists</h3>
+      </div>
+    </div>
   </body>
 </html>

web/web.go

@@ -7,7 +7,9 @@ import (
 	"net/http"
 	"os"
 	"strconv"
-	"muzi/importsongs"
+
+	"muzi/migrate"
+
 	"golang.org/x/crypto/bcrypt"
 
 	"github.com/go-chi/chi/v5"
@@ -24,7 +26,6 @@ type PageData struct {
 	Page    int
 }
 
-
 func Sub(a int, b int) int {
 	return a - b
 }
@@ -35,7 +36,12 @@ func Add(a int, b int) int {
 
 func getTimes(conn *pgx.Conn, lim int, off int) []string {
 	var times []string
-	rows, err := conn.Query(context.Background(), "SELECT timestamp FROM history ORDER BY timestamp DESC LIMIT $1 OFFSET $2;", lim, off)
+	rows, err := conn.Query(
+		context.Background(),
+		"SELECT timestamp FROM history ORDER BY timestamp DESC LIMIT $1 OFFSET $2;",
+		lim,
+		off,
+	)
 	if err != nil {
 		fmt.Fprintf(os.Stderr, "SELECT COUNT failed: %v\n", err)
 		return nil
@@ -54,7 +60,12 @@ func getTimes(conn *pgx.Conn, lim int, off int) []string {
 
 func getTitles(conn *pgx.Conn, lim int, off int) []string {
 	var titles []string
-	rows, err := conn.Query(context.Background(), "SELECT song_name FROM history ORDER BY timestamp DESC LIMIT $1 OFFSET $2;", lim, off)
+	rows, err := conn.Query(
+		context.Background(),
+		"SELECT song_name FROM history ORDER BY timestamp DESC LIMIT $1 OFFSET $2;",
+		lim,
+		off,
+	)
 	if err != nil {
 		fmt.Fprintf(os.Stderr, "SELECT COUNT failed: %v\n", err)
 		return nil
@@ -73,7 +84,12 @@ func getTitles(conn *pgx.Conn, lim int, off int) []string {
 
 func getArtists(conn *pgx.Conn, lim int, off int) []string {
 	var artists []string
-	rows, err := conn.Query(context.Background(), "SELECT artist FROM history ORDER BY timestamp DESC LIMIT $1 OFFSET $2;", lim, off)
+	rows, err := conn.Query(
+		context.Background(),
+		"SELECT artist FROM history ORDER BY timestamp DESC LIMIT $1 OFFSET $2;",
+		lim,
+		off,
+	)
 	if err != nil {
 		fmt.Fprintf(os.Stderr, "SELECT COUNT failed: %v\n", err)
 		return nil
@@ -118,7 +134,10 @@ func verifyPassword(hashedPassword string, enteredPassword []byte) bool {
 }
 
 func createAccount(w http.ResponseWriter, r *http.Request) {
-	conn, err := pgx.Connect(context.Background(), "postgres://postgres:postgres@localhost:5432/muzi")
+	conn, err := pgx.Connect(
+		context.Background(),
+		"postgres://postgres:postgres@localhost:5432/muzi",
+	)
 	if err != nil {
 		fmt.Fprintf(os.Stderr, "Cannot connect to muzi database: %v\n", err)
 		return
@@ -131,10 +150,17 @@ func createAccount(w http.ResponseWriter, r *http.Request) {
 		username := r.FormValue("uname")
 		hashedPassword := hashPassword([]byte(r.FormValue("pass")))
 
-	if importsongs.TableExists("users", conn) == false {
+		if !migrate.TableExists("users", conn) {
 			_, err = conn.Exec(
 				context.Background(),
-			`CREATE TABLE users (username TEXT, password TEXT, pk SERIAL, PRIMARY KEY (pk));`,
+				`CREATE TABLE users (
+					username TEXT,
+					password TEXT,
+					bio TEXT,
+					pfp TEXT,
+					pk SERIAL,
+					PRIMARY KEY (pk)
+				);`,
 			)
 			if err != nil {
 				fmt.Fprintf(os.Stderr, "Cannot create users table: %v\n", err)
@@ -143,35 +169,43 @@ func createAccount(w http.ResponseWriter, r *http.Request) {
 		}
 
 		_, err = conn.Exec(
-				context.Background(), `INSERT INTO users (username, password) VALUES ($1, $2);`,
+			context.Background(),
+			`INSERT INTO users (username, password, bio, pfp) VALUES ($1, $2, $3, $4);`,
 			username,
 			hashedPassword,
+			"This profile has no bio.",
+			"/files/assets/default.png",
 		)
 		if err != nil {
 			fmt.Fprintf(os.Stderr, "Cannot add new user to users table: %v\n", err)
 			http.Redirect(w, r, "/createaccount", http.StatusSeeOther)
 		} else {
-			http.Redirect(w, r, "/profile/" + username, http.StatusSeeOther)
+			http.Redirect(w, r, "/profile/"+username, http.StatusSeeOther)
 		}
 	}
 }
 
-func createAccountHandler(w http.ResponseWriter, r *http.Request) {
+func createAccountPageHandler() http.HandlerFunc {
-	tmp, err := template.New("create_account.gohtml").ParseFiles("./templates/create_account.gohtml")
+	return func(w http.ResponseWriter, r *http.Request) {
+		tmp, err := template.New("create_account.gohtml").
+			ParseFiles("./templates/create_account.gohtml")
 		if err != nil {
 			http.Error(w, err.Error(), http.StatusInternalServerError)
 			return
 		}
-
 		err = tmp.Execute(w, nil)
 		if err != nil {
 			http.Error(w, err.Error(), http.StatusInternalServerError)
 			return
 		}
+	}
 }
 
 func loginSubmit(w http.ResponseWriter, r *http.Request) {
-	conn, err := pgx.Connect(context.Background(), "postgres://postgres:postgres@localhost:5432/muzi")
+	conn, err := pgx.Connect(
+		context.Background(),
+		"postgres://postgres:postgres@localhost:5432/muzi",
+	)
 	if err != nil {
 		fmt.Fprintf(os.Stderr, "Cannot connect to muzi database: %v\n", err)
 		return
@@ -184,34 +218,48 @@ func loginSubmit(w http.ResponseWriter, r *http.Request) {
 		username := r.FormValue("uname")
 		password := r.FormValue("pass")
 		var storedPassword string
-		err := conn.QueryRow(context.Background(), "SELECT password FROM users WHERE username = $1;", username).Scan(&storedPassword)
+		err := conn.QueryRow(context.Background(), "SELECT password FROM users WHERE username = $1;", username).
+			Scan(&storedPassword)
 		if err != nil {
 			fmt.Fprintf(os.Stderr, "Cannot get password for entered username: %v\n", err)
 		}
 
 		if verifyPassword(storedPassword, []byte(password)) {
-			http.Redirect(w, r, "/profile/" + username, http.StatusSeeOther)
+			http.Redirect(w, r, "/profile/"+username, http.StatusSeeOther)
+		} else {
+			http.Redirect(w, r, "/login?error=1", http.StatusSeeOther)
 		}
 	}
 }
 
-func loginPage(w http.ResponseWriter, r *http.Request) {
+func loginPageHandler() http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		type data struct {
+			ShowError bool
+		}
+		d := data{ShowError: false}
+		if r.URL.Query().Get("error") != "" {
+			d.ShowError = true
+		}
 		tmp, err := template.New("login.gohtml").ParseFiles("./templates/login.gohtml")
 		if err != nil {
 			http.Error(w, err.Error(), http.StatusInternalServerError)
 			return
 		}
 
-	err = tmp.Execute(w, nil)
+		err = tmp.Execute(w, d)
 		if err != nil {
 			http.Error(w, err.Error(), http.StatusInternalServerError)
 			return
 		}
+	}
 }
 
 func historyPage(w http.ResponseWriter, r *http.Request) {
-
+	conn, err := pgx.Connect(
-	conn, err := pgx.Connect(context.Background(), "postgres://postgres:postgres@localhost:5432/muzi")
+		context.Background(),
+		"postgres://postgres:postgres@localhost:5432/muzi",
+	)
 	if err != nil {
 		fmt.Fprintf(os.Stderr, "Cannot connect to muzi database: %v\n", err)
 		return
@@ -242,12 +290,14 @@ func historyPage(w http.ResponseWriter, r *http.Request) {
 		Page:    pageInt,
 	}
 
-funcMap := template.FuncMap{
+	funcMap := template.FuncMap{
 		"Sub": Sub,
 		"Add": Add,
-}
+	}
 
-	tmp, err := template.New("history.gohtml").Funcs(funcMap).ParseFiles("./templates/history.gohtml")
+	tmp, err := template.New("history.gohtml").
+		Funcs(funcMap).
+		ParseFiles("./templates/history.gohtml")
 	if err != nil {
 		http.Error(w, err.Error(), http.StatusInternalServerError)
 		return
@@ -260,29 +310,34 @@ funcMap := template.FuncMap{
 	}
 }
 
-type Profile struct {
+func profilePageHandler() http.HandlerFunc {
-	Username string
+	return func(w http.ResponseWriter, r *http.Request) {
-	Bio string
-}
-
-func Start() {
-	addr := ":1234"
-	r := chi.NewRouter()
-	r.Use(middleware.Logger)
-	r.Get("/static/style.css", func(w http.ResponseWriter, r *http.Request) {
-		http.ServeFile(w, r, "./static/style.css")
-	})
-	r.Get("/history", historyPage)
-	r.Get("/login", loginPage)
-	r.Get("/createaccount", createAccountHandler)
-	// TODO: clean this up
-	r.Get("/profile/{username}", func(w http.ResponseWriter, r *http.Request) {
 		username := chi.URLParam(r, "username")
 
-		profileData := Profile {
+		conn, err := pgx.Connect(
-			Username: username,
+			context.Background(),
-			Bio: "default",
+			"postgres://postgres:postgres@localhost:5432/muzi",
+		)
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "Cannot connect to muzi database: %v\n", err)
+			http.Error(w, err.Error(), http.StatusInternalServerError)
+			return
 		}
+		defer conn.Close(context.Background())
+
+		var profileData Profile
+
+		err = conn.QueryRow(
+			context.Background(),
+			"SELECT bio, pfp FROM users WHERE username = $1;",
+			username,
+		).Scan(&profileData.Bio, &profileData.Pfp)
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "Cannot get profile for %s: %v\n", username, err)
+			http.Error(w, err.Error(), http.StatusInternalServerError)
+			return
+		}
+		profileData.Username = username
 
 		tmp, err := template.ParseFiles("./templates/profile.gohtml")
 		if err != nil {
@@ -290,7 +345,24 @@ func Start() {
 			return
 		}
 		tmp.Execute(w, profileData)
-	})
+	}
+}
+
+type Profile struct {
+	Username string
+	Bio      string
+	Pfp      string
+}
+
+func Start() {
+	addr := ":1234"
+	r := chi.NewRouter()
+	r.Use(middleware.Logger)
+	r.Handle("/files/*", http.StripPrefix("/files", http.FileServer(http.Dir("./static"))))
+	r.Get("/history", historyPage)
+	r.Get("/login", loginPageHandler())
+	r.Get("/createaccount", createAccountPageHandler())
+	r.Get("/profile/{username}", profilePageHandler())
 	r.Post("/loginsubmit", loginSubmit)
 	r.Post("/createaccountsubmit", createAccount)
 	fmt.Printf("WebUI starting on %s\n", addr)