mirror of
https://github.com/riwiwa/muzi.git
synced 2026-02-28 11:56:57 -08:00
437 lines
11 KiB
Go
437 lines
11 KiB
Go
package importsongs
|
|
|
|
import (
|
|
"archive/zip"
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"io"
|
|
"net/http"
|
|
"os"
|
|
"path/filepath"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/jackc/pgx/v5"
|
|
)
|
|
|
|
const (
|
|
spotify = iota
|
|
lastfm
|
|
apple
|
|
)
|
|
|
|
func TableExists(name string, conn *pgx.Conn) bool {
|
|
var exists bool
|
|
err := conn.QueryRow(
|
|
context.Background(),
|
|
`SELECT EXISTS (SELECT 1 FROM pg_tables WHERE schemaname = 'public' AND
|
|
tablename = $1);`,
|
|
name,
|
|
).
|
|
Scan(&exists)
|
|
if err != nil {
|
|
fmt.Fprintf(os.Stderr, "SELECT EXISTS failed: %v\n", err)
|
|
return false
|
|
}
|
|
return exists
|
|
}
|
|
|
|
func DbExists() bool {
|
|
conn, err := pgx.Connect(
|
|
context.Background(),
|
|
"postgres://postgres:postgres@localhost:5432/muzi",
|
|
)
|
|
if err != nil {
|
|
return false
|
|
}
|
|
defer conn.Close(context.Background())
|
|
return true
|
|
}
|
|
|
|
func CreateDB() error {
|
|
conn, err := pgx.Connect(
|
|
context.Background(),
|
|
"postgres://postgres:postgres@localhost:5432",
|
|
)
|
|
if err != nil {
|
|
fmt.Fprintf(os.Stderr, "Cannot connect to PostgreSQL: %v\n", err)
|
|
return err
|
|
}
|
|
defer conn.Close(context.Background())
|
|
_, err = conn.Exec(context.Background(), "CREATE DATABASE muzi")
|
|
if err != nil {
|
|
fmt.Fprintf(os.Stderr, "Cannot create muzi database: %v\n", err)
|
|
return err
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func JsonToDB(jsonFile string, platform int) error {
|
|
if !DbExists() {
|
|
err := CreateDB()
|
|
if err != nil {
|
|
fmt.Fprintf(os.Stderr, "Error creating muzi database: %v\n", err)
|
|
panic(err)
|
|
}
|
|
}
|
|
conn, err := pgx.Connect(
|
|
context.Background(),
|
|
"postgres://postgres:postgres@localhost:5432/muzi",
|
|
)
|
|
if err != nil {
|
|
fmt.Fprintf(os.Stderr, "Cannot connect to muzi database: %v\n", err)
|
|
panic(err)
|
|
}
|
|
defer conn.Close(context.Background())
|
|
if !TableExists("history", conn) {
|
|
_, err = conn.Exec(
|
|
context.Background(),
|
|
`CREATE TABLE history ( ms_played INTEGER, timestamp TIMESTAMPTZ,
|
|
song_name TEXT, artist TEXT, album_name TEXT, PRIMARY KEY (timestamp,
|
|
ms_played, artist, song_name));`,
|
|
)
|
|
}
|
|
if err != nil {
|
|
fmt.Fprintf(os.Stderr, "Cannot create history table: %v\n", err)
|
|
panic(err)
|
|
}
|
|
jsonData, err := os.ReadFile(jsonFile)
|
|
if err != nil {
|
|
fmt.Fprintf(os.Stderr, "Cannot read %s: %v\n", jsonFile, err)
|
|
return err
|
|
}
|
|
if platform == spotify {
|
|
type Track struct {
|
|
Timestamp string `json:"ts"`
|
|
Platform string `json:"-"`
|
|
Played int `json:"ms_played"`
|
|
Country string `json:"-"`
|
|
IP string `json:"-"`
|
|
Name string `json:"master_metadata_track_name"`
|
|
Artist string `json:"master_metadata_album_artist_name"`
|
|
Album string `json:"master_metadata_album_album_name"`
|
|
TrackURI string `json:"-"`
|
|
Episode string `json:"-"`
|
|
Show string `json:"-"`
|
|
EpisodeURI string `json:"-"`
|
|
Audiobook string `json:"-"`
|
|
AudiobookURI string `json:"-"`
|
|
AudiobookChapterURI string `json:"-"`
|
|
AudiobookChapter string `json:"-"`
|
|
ReasonStart string `json:"-"`
|
|
ReasonEnd string `json:"-"`
|
|
Shuffle bool `json:"-"`
|
|
Skipped bool `json:"-"`
|
|
Offline bool `json:"-"`
|
|
OfflineTimestamp int `json:"-"`
|
|
Incognito bool `json:"-"`
|
|
}
|
|
var tracks []Track
|
|
err := json.Unmarshal(jsonData, &tracks)
|
|
if err != nil {
|
|
fmt.Fprintf(os.Stderr, "Cannot unmarshal %s: %v\n", jsonFile, err)
|
|
return err
|
|
}
|
|
for _, track := range tracks {
|
|
// skip adding a song if it was only listed to for less than 20 seconds
|
|
if track.Played < 20000 {
|
|
continue
|
|
}
|
|
_, err = conn.Exec(
|
|
context.Background(),
|
|
`INSERT INTO history (timestamp, song_name, artist, album_name,
|
|
ms_played) VALUES ($1, $2, $3, $4, $5);`,
|
|
track.Timestamp,
|
|
track.Name,
|
|
track.Artist,
|
|
track.Album,
|
|
track.Played,
|
|
)
|
|
if err != nil {
|
|
fmt.Fprintf(
|
|
os.Stderr,
|
|
"Couldn't add track to muzi DB (%s): %v\n",
|
|
(track.Artist + " - " + track.Name),
|
|
err,
|
|
)
|
|
}
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func AddDirToDB(path string, platform int) error {
|
|
dirs, err := os.ReadDir(path)
|
|
if err != nil {
|
|
fmt.Fprintf(os.Stderr, "Error while reading path: %s: %v\n", path, err)
|
|
return err
|
|
}
|
|
for _, dir := range dirs {
|
|
subPath := filepath.Join(
|
|
path,
|
|
dir.Name(),
|
|
"Spotify Extended Streaming History",
|
|
)
|
|
entries, err := os.ReadDir(subPath)
|
|
if err != nil {
|
|
fmt.Fprintf(os.Stderr, "Error while reading path: %s: %v\n", subPath, err)
|
|
return err
|
|
}
|
|
for _, f := range entries {
|
|
jsonFileName := f.Name()
|
|
if platform == spotify {
|
|
if !strings.Contains(jsonFileName, ".json") {
|
|
continue
|
|
}
|
|
// prevents parsing spotify video data that causes duplicates
|
|
if strings.Contains(jsonFileName, "Video") {
|
|
continue
|
|
}
|
|
}
|
|
jsonFilePath := filepath.Join(subPath, jsonFileName)
|
|
err = JsonToDB(jsonFilePath, platform)
|
|
if err != nil {
|
|
fmt.Fprintf(os.Stderr,
|
|
"Error adding json data (%s) to muzi database: %v", jsonFilePath, err)
|
|
return err
|
|
}
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func ImportLastFM(username string, apiKey string) error {
|
|
if !DbExists() {
|
|
err := CreateDB()
|
|
if err != nil {
|
|
fmt.Fprintf(os.Stderr, "Error creating muzi database: %v\n", err)
|
|
panic(err)
|
|
}
|
|
}
|
|
conn, err := pgx.Connect(
|
|
context.Background(),
|
|
"postgres://postgres:postgres@localhost:5432/muzi",
|
|
)
|
|
if err != nil {
|
|
fmt.Fprintf(os.Stderr, "Cannot connect to muzi database: %v\n", err)
|
|
panic(err)
|
|
}
|
|
defer conn.Close(context.Background())
|
|
if !TableExists("history", conn) {
|
|
_, err = conn.Exec(
|
|
context.Background(),
|
|
`CREATE TABLE history ( ms_played INTEGER, timestamp TIMESTAMPTZ,
|
|
song_name TEXT, artist TEXT, album_name TEXT, PRIMARY KEY (timestamp,
|
|
ms_played, artist, song_name));`,
|
|
)
|
|
}
|
|
if err != nil {
|
|
fmt.Fprintf(os.Stderr, "Cannot create history table: %v\n", err)
|
|
panic(err)
|
|
}
|
|
|
|
resp, err := http.Get(
|
|
"https://ws.audioscrobbler.com/2.0/?method=user.getrecenttracks&user=" +
|
|
username + "&api_key=" + apiKey + "&format=json&limit=1",
|
|
)
|
|
if err != nil {
|
|
fmt.Fprintf(os.Stderr, "Error getting LastFM http response: %v\n", err)
|
|
return err
|
|
}
|
|
type Response struct {
|
|
Recenttracks struct {
|
|
Track []struct {
|
|
Artist struct {
|
|
Mbid string `json:"-"`
|
|
Text string `json:"#text"`
|
|
} `json:"artist"`
|
|
Streamable string `json:"-"`
|
|
Image []struct {
|
|
Size string `json:"-"`
|
|
Text string `json:"-"`
|
|
} `json:"-"`
|
|
Mbid string `json:"-"`
|
|
Album struct {
|
|
Mbid string `json:"-"`
|
|
Text string `json:"#text"`
|
|
} `json:"album"`
|
|
Name string `json:"name"`
|
|
Attr struct {
|
|
Nowplaying string `json:"nowplaying"`
|
|
} `json:"@attr,omitempty"`
|
|
URL string `json:"-"`
|
|
Date struct {
|
|
Uts string `json:"uts"`
|
|
Text string `json:"-"`
|
|
} `json:"date"`
|
|
} `json:"track"`
|
|
Attr struct {
|
|
PerPage string `json:"-"`
|
|
TotalPages string `json:"totalPages"`
|
|
Page string `json:"page"`
|
|
Total string `json:"-"`
|
|
User string `json:"-"`
|
|
} `json:"@attr"`
|
|
} `json:"recenttracks"`
|
|
}
|
|
var data Response
|
|
json.NewDecoder(resp.Body).Decode(&data)
|
|
totalPages, err := strconv.Atoi(data.Recenttracks.Attr.TotalPages)
|
|
if totalPages%100 != 0 {
|
|
totalPages = totalPages / 100
|
|
totalPages++
|
|
} else {
|
|
totalPages = totalPages / 100
|
|
}
|
|
|
|
for i := 1; i <= totalPages; i++ {
|
|
resp, err := http.Get(
|
|
"https://ws.audioscrobbler.com/2.0/?method=user.getrecenttracks&user=" +
|
|
username + "&api_key=" + apiKey + "&format=json&limit=100&page=" +
|
|
strconv.Itoa(
|
|
i,
|
|
),
|
|
)
|
|
if err != nil {
|
|
fmt.Fprintf(os.Stderr, "Error getting LastFM http response: %v\n", err)
|
|
return err
|
|
}
|
|
json.NewDecoder(resp.Body).Decode(&data)
|
|
for j := range data.Recenttracks.Track {
|
|
if data.Recenttracks.Track[j].Attr.Nowplaying == "true" {
|
|
continue
|
|
}
|
|
unixTime, err := strconv.ParseInt(
|
|
data.Recenttracks.Track[j].Date.Uts,
|
|
10,
|
|
64,
|
|
)
|
|
if err != nil {
|
|
fmt.Fprintf(os.Stderr, "Error parsing string for int: %v\n", err)
|
|
return err
|
|
}
|
|
ts := time.Unix(unixTime, 0)
|
|
_, err = conn.Exec(
|
|
context.Background(),
|
|
`INSERT INTO history (timestamp, song_name, artist, album_name,
|
|
ms_played) VALUES ($1, $2, $3, $4, $5);`,
|
|
ts,
|
|
data.Recenttracks.Track[j].Name,
|
|
data.Recenttracks.Track[j].Artist.Text,
|
|
data.Recenttracks.Track[j].Album.Text,
|
|
0,
|
|
)
|
|
if err != nil {
|
|
fmt.Fprintf(
|
|
os.Stderr,
|
|
"Couldn't add track to muzi DB (%s): %v\n",
|
|
(data.Recenttracks.Track[j].Artist.Text + " - " +
|
|
data.Recenttracks.Track[j].Name), err)
|
|
}
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func ImportSpotify() error {
|
|
path := filepath.Join(".", "imports", "spotify", "zip")
|
|
targetBase := filepath.Join(".", "imports", "spotify", "extracted")
|
|
entries, err := os.ReadDir(path)
|
|
if err != nil {
|
|
fmt.Fprintf(os.Stderr, "Error reading path: %s: %v\n", path, err)
|
|
return err
|
|
}
|
|
for _, f := range entries {
|
|
_, err := zip.OpenReader(filepath.Join(path, f.Name()))
|
|
if err != nil {
|
|
fmt.Fprintf(os.Stderr, "Error opening zip: %s: %v\n",
|
|
filepath.Join(path, f.Name()), err)
|
|
continue
|
|
}
|
|
fileName := f.Name()
|
|
fileFullPath := filepath.Join(path, fileName)
|
|
fileBaseName := fileName[:(strings.LastIndex(fileName, "."))]
|
|
targetDirFullPath := filepath.Join(targetBase, fileBaseName)
|
|
|
|
err = Extract(fileFullPath, targetDirFullPath)
|
|
if err != nil {
|
|
fmt.Fprintf(os.Stderr, "Error extracting %s to %s: %v\n",
|
|
fileFullPath, targetDirFullPath, err)
|
|
return err
|
|
}
|
|
}
|
|
err = AddDirToDB(targetBase, spotify)
|
|
if err != nil {
|
|
fmt.Fprintf(os.Stderr,
|
|
"Error adding directory of data (%s) to muzi database: %v\n",
|
|
targetBase, err)
|
|
return err
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func Extract(path string, target string) error {
|
|
archive, err := zip.OpenReader(path)
|
|
if err != nil {
|
|
fmt.Fprintf(os.Stderr, "Error opening zip: %s: %v\n", path, err)
|
|
return err
|
|
}
|
|
defer archive.Close()
|
|
|
|
zipDir := filepath.Base(path)
|
|
zipDir = zipDir[:(strings.LastIndex(zipDir, "."))]
|
|
|
|
for _, f := range archive.File {
|
|
filePath := filepath.Join(target, f.Name)
|
|
fmt.Println("extracting:", filePath)
|
|
|
|
if !strings.HasPrefix(
|
|
filePath,
|
|
filepath.Clean(target)+string(os.PathSeparator),
|
|
) {
|
|
err = fmt.Errorf("Invalid file path: %s", filePath)
|
|
fmt.Fprintf(os.Stderr, "%v\n", err)
|
|
return err
|
|
}
|
|
if f.FileInfo().IsDir() {
|
|
fmt.Println("Creating Directory", filePath)
|
|
os.MkdirAll(filePath, os.ModePerm)
|
|
continue
|
|
}
|
|
if err := os.MkdirAll(filepath.Dir(filePath), os.ModePerm); err != nil {
|
|
fmt.Fprintf(os.Stderr, "Error making directory: %s: %v\n",
|
|
filepath.Dir(filePath), err)
|
|
return err
|
|
}
|
|
fileToExtract, err := os.OpenFile(
|
|
filePath,
|
|
os.O_WRONLY|os.O_CREATE|os.O_TRUNC,
|
|
f.Mode(),
|
|
)
|
|
if err != nil {
|
|
fmt.Fprintf(os.Stderr, "Error opening file: %s: %v\n", filePath, err)
|
|
return err
|
|
}
|
|
extractedFile, err := f.Open()
|
|
if err != nil {
|
|
fmt.Fprintf(os.Stderr, "Error opening file: %s: %v\n", f.Name, err)
|
|
return err
|
|
}
|
|
if _, err := io.Copy(fileToExtract, extractedFile); err != nil {
|
|
fmt.Fprintf(
|
|
os.Stderr,
|
|
"Error while copying file: %s to: %s: %v\n",
|
|
fileToExtract.Name(),
|
|
extractedFile,
|
|
err,
|
|
)
|
|
return err
|
|
}
|
|
fileToExtract.Close()
|
|
extractedFile.Close()
|
|
}
|
|
return nil
|
|
}
|