Switched YouTube backend to Innertube API

This commit is contained in:
Luxferre
2024-09-18 18:57:23 +03:00
parent 70aceaa006
commit e40807f4d7
7 changed files with 282 additions and 158 deletions
+2 -1
View File
@@ -11,5 +11,6 @@ require (
github.com/EDDYCJY/fake-useragent v0.2.0 // indirect
github.com/PuerkitoBio/goquery v1.7.1 // indirect
github.com/andybalholm/cascadia v1.3.1 // indirect
golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f // indirect
github.com/stretchr/testify v1.9.0 // indirect
golang.org/x/net v0.29.0 // indirect
)
+8 -8
View File
@@ -9,21 +9,21 @@ github.com/PuerkitoBio/goquery v1.7.1/go.mod h1:XY0pP4kfraEmmV1O7Uf6XyjoslwsneBb
github.com/andybalholm/cascadia v1.2.0/go.mod h1:YCyR8vOZT9aZ1CHEd8ap0gMVm2aFgxBp0T0eFw1RUQY=
github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c=
github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA=
github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f h1:OfiFi4JbukWwe3lzw+xunroH1mnC1e2Gy5cxNJApiSY=
golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.29.0 h1:5ORfpBpCs4HzDYoodCDBbwHzdR5UrLBZ3sOnUJmFoHo=
golang.org/x/net v0.29.0/go.mod h1:gLkgy8jTGERgjzMic6DS9+SP0ajcu6Xu3Orq/SpETg0=
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+237
View File
@@ -0,0 +1,237 @@
/*
StreamGoose YouTube backend — Innertube API
Created by Luxferre in 2024, released into public domain
*/
package main
import (
"fmt"
"log"
"bytes"
"strings"
"strconv"
"regexp"
"time"
"encoding/json"
"net/http"
"io/ioutil"
)
var YT_INNER_USERAGENT string
/* InnerTube chat options structure */
type YTIChatOptions struct {
Key string `json:"key"` /* InnerTube API key */
Continuation string `json:"continuation"` /* continuation key */
ClientName string `json:"clientName"` /* client name string */
ClientVersion string `json:"clientVersion"` /* client version string */
}
/* API request helper functions */
func yt_inner_api_req(url string, data []byte) string {
method := "GET"
if len(data) > 0 {
method = "POST"
}
req, err := http.NewRequest(method, url, bytes.NewReader(data))
req.Header.Set("User-Agent", YT_INNER_USERAGENT)
req.Header.Set("Accept-Language", "en-US")
client := &http.Client{}
resp, err := client.Do(req)
if err != nil {
log.Fatal(err)
}
body, _ := ioutil.ReadAll(resp.Body)
resp.Body.Close()
/* print the response if the debug flag is set */
if YT_DEBUG {
log.Printf("[youtube] API response: %s", string(body))
}
return string(body)
}
/* convert a series of runs into a text message */
func yt_inner_runs_to_text(runs []interface{}) string {
out_text := ""
for _, rrun := range runs { /* iterate over the runs */
run := rrun.(map[string]interface{})
if _, ok := run["text"]; ok { /* normal text */
out_text += run["text"].(string)
} else if _, ok := run["emoji"]; ok { /* this is where the fun begins */
emoji := run["emoji"].(map[string]interface{})
if _, ok := emoji["isCustomEmoji"]; ok && emoji["isCustomEmoji"].(bool) == true { /* custom emoji */
/* TODO: out_text += " #YTEMOTE-" + emoji["image"]["thumbnails"][0]["url"] + "# " */
if len(emoji["shortcuts"].([]interface{})) > 0 {
out_text += " " + emoji["shortcuts"].([]interface{})[0].(string) + " "
}
} else { /* standard Unicode emoji, pass as text */
out_text += emoji["emojiId"].(string)
}
}
}
return out_text
}
/* convert inner chat action into a public chat item */
func yt_inner_action_to_chatitem(action map[string]interface{}) ChatMessage {
item := action["addChatItemAction"].(map[string]interface{})["item"].(map[string]interface{})
/* get the renderer */
var renderer map[string]interface{}
if _, ok := item["liveChatTextMessageRenderer"]; ok {
renderer = item["liveChatTextMessageRenderer"].(map[string]interface{})
} else if _, ok := item["liveChatPaidMessageRenderer"]; ok {
renderer = item["liveChatPaidMessageRenderer"].(map[string]interface{})
} else if _, ok := item["liveChatPaidStickerRenderer"]; ok {
renderer = item["liveChatPaidStickerRenderer"].(map[string]interface{})
} else if _, ok := item["liveChatMembershipItemRenderer"]; ok {
renderer = item["liveChatMembershipItemRenderer"].(map[string]interface{})
} else {
renderer = nil
}
if renderer == nil { /* invalid renderer field */
return ChatMessage{}
}
/* here, message text is composed as a series of individual runs */
var mruns []interface{}
if _, ok := renderer["message"]; ok {
mruns = renderer["message"].(map[string]interface{})["runs"].([]interface{})
} else if _, ok := renderer["headerSubtext"]; ok {
mruns = renderer["headerSubtext"].(map[string]interface{})["runs"].([]interface{})
}
/* fetch the author display name */
author_name := ""
if _, ok := renderer["authorName"]; ok {
author_name = renderer["authorName"].(map[string]interface{})["simpleText"].(string)
}
/* fetch the author icon URL */
author_icon := ""
if _, ok := renderer["authorPhoto"]; ok {
thumbnails := renderer["authorPhoto"].(map[string]interface{})["thumbnails"].([]interface{})
if len(thumbnails) > 0 { /* the first thumbnail URL will do */
author_icon = thumbnails[0].(map[string]interface{})["url"].(string)
}
}
/* fetch the timestamp (microseconds) */
ts, err := strconv.ParseInt(renderer["timestampUsec"].(string), 10, 0)
if err != nil { /* fallback to current time if anything */
ts = time.Now().UnixMicro()
}
sent_dt := time.UnixMicro(ts)
/* prepare output message */
out_message := ChatMessage {
Type: "yt",
Username: author_name,
Timestamp: sent_dt.UTC().Format("2006-01-02T15:04:05Z"),
Text: yt_inner_runs_to_text(mruns),
Icon: author_icon,
}
/* check roles */
if _, ok := renderer["authorBadges"]; ok {
for _, entry := range renderer["authorBadges"].([]interface{}) {
badge := entry.(map[string]interface{})["liveChatAuthorBadgeRenderer"].(map[string]interface{})
if _, ok := badge["customThumbnail"]; !ok { /* custom badges not supported yet */
switch badge["icon"].(map[string]interface{})["iconType"].(string) {
case "OWNER":
out_message.Type += "-broadcaster"
break
case "MODERATOR":
out_message.Type += "-moderator"
break
case "VERIFIED":
out_message.Type += "-verified"
break
}
}
}
}
if _, ok := renderer["purchaseAmountText"]; ok {
out_message.Type += "-sponsor"
}
return out_message
}
/* fetch chat options by the broadcast video ID or handle */
func yt_inner_get_chat_options(video_id string) YTIChatOptions {
var opts YTIChatOptions
var live_url string
if strings.HasPrefix(video_id, "@") { /* it's a handle */
live_url = fmt.Sprintf("https://www.youtube.com/%s/live", video_id)
} else { /* it's a broadcast video ID */
live_url = fmt.Sprintf("https://www.youtube.com/live_chat?v=%s", video_id)
}
raw_data := yt_inner_api_req(live_url, nil)
opts.ClientName = "WEB"
r, _ := regexp.Compile("\"INNERTUBE_API_KEY\":\"(.+?)\"")
rmatch := r.FindStringSubmatch(raw_data)
if len(rmatch) > 0 {
opts.Key = rmatch[1]
} else {
log.Fatal("[youtube] Cannot connect to the YouTube livechat (innertube key)")
}
r, _ = regexp.Compile("\"continuation\":\"(.+?)\"")
rmatch = r.FindStringSubmatch(raw_data)
if len(rmatch) > 0 {
opts.Continuation = rmatch[1]
} else {
log.Fatal("[youtube] Cannot connect to the YouTube livechat (continuation)")
}
r, _ = regexp.Compile("\"clientVersion\":\"(.+?)\"")
rmatch = r.FindStringSubmatch(raw_data)
if len(rmatch) > 0 {
opts.ClientVersion = rmatch[1]
} else {
log.Fatal("[youtube] Cannot connect to the YouTube livechat (clientversion)")
}
return opts
}
/* fetch chat data using the chat options */
/* return the list of ChatMessage items */
/* should update the options structure */
func yt_inner_get_chat_data(opts *YTIChatOptions) []ChatMessage {
jsondata := make(map[string]interface{})
jsondata["context"] = make(map[string]interface{})
jsondata["continuation"] = opts.Continuation
jsondata["context"].(map[string]interface{})["client"] = make(map[string]string)
jsondata["context"].(map[string]interface{})["client"].(map[string]string)["userAgent"] = YT_INNER_USERAGENT
jsondata["context"].(map[string]interface{})["client"].(map[string]string)["clientName"] = opts.ClientName
jsondata["context"].(map[string]interface{})["client"].(map[string]string)["clientVersion"] = opts.ClientVersion
jsonstr, _ := json.Marshal(&jsondata)
/* post the marshalled string and get the result into chatdata */
target_url := fmt.Sprintf("https://www.youtube.com/youtubei/v1/live_chat/get_live_chat?key=%s", opts.Key)
rawresp := yt_inner_api_req(target_url, jsonstr)
var chatdata map[string]interface{}
err := json.Unmarshal([]byte(rawresp), &chatdata)
if err != nil {
log.Fatal(err)
}
var chatitems []ChatMessage
/* process the upper structure */
if _, ok := chatdata["continuationContents"]; ok {
cc := chatdata["continuationContents"].(map[string]interface{})
if _, ok := cc["liveChatContinuation"]; ok {
lcc := cc["liveChatContinuation"].(map[string]interface{})
if lcc["actions"] != nil {
actions := lcc["actions"].([]interface{})
conts := lcc["continuations"].([]interface{})
icd := conts[0].(map[string]interface{})["invalidationContinuationData"].(map[string]interface{})
opts.Continuation = icd["continuation"].(string)
for _, action := range actions { /* iterate through the actions */
if _, ok := action.(map[string]interface{})["addChatItemAction"]; ok {
chatitem := yt_inner_action_to_chatitem(action.(map[string]interface{}))
if chatitem.Type != "" {
chatitems = append(chatitems, chatitem)
}
}
}
}
}
}
return chatitems
}
+25 -134
View File
@@ -10,165 +10,56 @@ import (
"log"
"slices"
"time"
"encoding/json"
"net/http"
"io/ioutil"
)
/* global API root URL placeholder */
var YT_API_ROOT string
const YT_ID_CACHE_LIMIT = 1000
/* global message ID cache */
var yt_msg_id_cache []string
/* API request helper function */
func yt_api_get(endpoint string) map[string]interface{} {
url := YT_API_ROOT + endpoint /* get full URL */
req, err := http.NewRequest("GET", url, nil)
req.Header.Set("Accept", "application/json; charset=utf-8")
client := &http.Client{}
resp, err := client.Do(req)
if err != nil {
log.Fatal(err)
}
body, _ := ioutil.ReadAll(resp.Body)
resp.Body.Close()
/* unmarshal the response */
var respbody map[string]interface{}
err = json.Unmarshal(body, &respbody)
if err != nil {
log.Fatal(err)
}
return respbody
}
/* main message procesing function */
func yt_process_message(rawmsg map[string]interface{}) {
/* if the message came from a real user, this is the one we should process */
if rawmsg["kind"] == "youtube#liveChatMessage" && rawmsg["snippet"].(map[string]interface{})["type"] == "textMessageEvent" {
if !slices.Contains(yt_msg_id_cache, rawmsg["id"].(string)) { /* we have a fresh message here */
snip := rawmsg["snippet"].(map[string]interface{})
author := rawmsg["authorDetails"].(map[string]interface{})
/* convert the timestamp */
sent_dt, _ := time.Parse("2006-01-02T15:04:05.000000-07:00", snip["publishedAt"].(string))
/* prepare output message */
out_message := ChatMessage {
Type: "yt",
Username: author["displayName"].(string),
Timestamp: sent_dt.UTC().Format("2006-01-02T15:04:05Z"),
Text: snip["displayMessage"].(string),
Icon: "",
}
/* check for profile image */
if _, ok := author["profileImageUrl"]; ok && len(author["profileImageUrl"].(string)) > 0 {
out_message.Icon = author["profileImageUrl"].(string)
}
/* check for verified role */
if _, ok := author["isVerified"]; ok && author["isVerified"].(bool) == true {
out_message.Type += "-verified"
}
/* check for broadcaster role */
if _, ok := author["isChatOwner"]; ok && author["isChatOwner"].(bool) == true {
out_message.Type += "-broadcaster"
}
/* check for moderator role */
if _, ok := author["isChatModerator"]; ok && author["isChatModerator"].(bool) == true {
out_message.Type += "-moderator"
}
/* check for sponsor role */
if _, ok := author["isChatSponsor"]; ok && author["isChatSponsor"].(bool) == true {
out_message.Type += "-sponsor"
}
/* finally, send it to the message broker via the message_transport channel */
message_transport <- out_message
/* update the ID cache */
yt_msg_id_cache = append(yt_msg_id_cache, rawmsg["id"].(string))
cachelen := len(yt_msg_id_cache)
if cachelen > YT_ID_CACHE_LIMIT { /* trim the excess */
yt_msg_id_cache = yt_msg_id_cache[cachelen-YT_ID_CACHE_LIMIT:cachelen]
}
}
}
}
/* debug flag */
var YT_DEBUG bool
/* entry point */
func youtube_run(config map[string]interface{}) {
API_POLL_TIMEOUT := 1 /* polling timeout in seconds */
if _, ok := config["yt_api_root_url"]; ok {
YT_API_ROOT = config["yt_api_root_url"].(string)
} else {
log.Fatal("[youtube] API Root URL missing in config!")
}
API_POLL_TIMEOUT := 2 /* polling timeout in seconds */
if _, ok := config["yt_api_poll_timeout"]; ok {
API_POLL_TIMEOUT = int(config["yt_api_poll_timeout"].(float64))
}
}
YT_INNER_USERAGENT = config["useragent"].(string)
YT_DEBUG = false
if _, ok := config["debug"]; ok {
YT_DEBUG = config["debug"].(bool)
}
video_id := "" /* video broadcast ID is empty by default */
if _, ok := config["yt_video_id"]; ok { /* override the video broadcast ID */
video_id = config["yt_video_id"].(string)
} else { /* no video ID supplied, begin with the usual flow */
/* first, get channel ID by YouTube handle */
channel_id_data := yt_api_get("/channels?part=id&forHandle=" + config["yt_channel"].(string))
channel_id := ""
if _, ok := channel_id_data["items"]; ok && len(channel_id_data["items"].([]interface{})) > 0 {
channel_id = channel_id_data["items"].([]interface{})[0].(map[string]interface{})["id"].(string)
} else {
log.Fatal("[youtube] Error while getting channel ID!")
}
/* then, get live broadcast video ID by channel ID */
video_id_data := yt_api_get("/search?part=snippet&eventType=live&type=video&channelId=" + channel_id)
if _, ok := video_id_data["items"]; ok && len(video_id_data["items"].([]interface{})) > 0 {
id_desc := video_id_data["items"].([]interface{})[0].(map[string]interface{})["id"]
video_id = id_desc.(map[string]interface{})["videoId"].(string)
} else {
log.Fatal("[youtube] Error while getting video ID!")
}
video_id = config["yt_channel"].(string)
}
/* now, get active live chat ID by broadcast video ID */
chat_id := ""
chat_id_data := yt_api_get("/videos?part=liveStreamingDetails&id=" + video_id)
if _, ok := chat_id_data["items"]; ok && len(chat_id_data["items"].([]interface{})) > 0 {
details := chat_id_data["items"].([]interface{})[0].(map[string]interface{})["liveStreamingDetails"]
chat_id = details.(map[string]interface{})["activeLiveChatId"].(string)
} else {
log.Fatal("[youtube] Error while getting live chat ID!")
}
if len(chat_id) < 1 { /* the chat ID is still empty */
log.Fatal("[youtube] Unknown error while getting live chat ID!")
}
opts := yt_inner_get_chat_options(video_id)
log.Println("StreamGoose YouTube Go backend started")
var raw_chat_data map[string]interface{}
yt_msg_id_cache = append(yt_msg_id_cache, "") /* init the cache */
for { /* main loop */
raw_chat_data = yt_api_get("/liveChat/messages?part=snippet,authorDetails&liveChatId=" + chat_id)
if _, ok := raw_chat_data["items"]; ok && len(raw_chat_data["items"].([]interface{})) > 0 {
for _, rawmsg := range raw_chat_data["items"].([]interface{}) { /* iterate over raw message objects */
yt_process_message(rawmsg.(map[string]interface{}))
chatitems := yt_inner_get_chat_data(&opts)
for _, msg := range chatitems {
if !slices.Contains(yt_msg_id_cache, msg.Timestamp + msg.Username) { /* we have a fresh message here */
/* send it to the message broker via the message_transport channel */
message_transport <- msg
/* update the ID cache */
yt_msg_id_cache = append(yt_msg_id_cache, msg.Timestamp + msg.Username)
cachelen := len(yt_msg_id_cache)
if cachelen > YT_ID_CACHE_LIMIT { /* trim the excess */
yt_msg_id_cache = yt_msg_id_cache[cachelen-YT_ID_CACHE_LIMIT:cachelen]
}
}
}
time.Sleep(time.Duration(API_POLL_TIMEOUT) * time.Second) /* wait for specified amount before polling next */
time.Sleep(time.Duration(API_POLL_TIMEOUT) * time.Second)
}
}