Release/debug and mojibake

.gitignore

@@ -30,6 +30,8 @@ replay_pid*
 */build/*
 build
 .vscode/*
+.cursor/*
 
 application.yml
-playerconfigs/*
+playerconfigs/*
+gradle.properties

README.md

@@ -20,6 +20,8 @@ Which clients are used is entirely configurable.
   - Information on using a `poToken` with `youtube-source`.
 - [Using a remote cipher server](#using-a-remote-cipher-server)
   - Information on using a remote cipher server with `youtube-source`.
+- [Debugging / Saving raw responses](#debugging--saving-raw-responses)
+  - Information on saving HTTP request/response data for debugging.
 - [REST Routes (`plugin` only)](#rest-routes-plugin-only)
   - Information on the REST routes provided by the `youtube-source` plugin module.
 - [Migration Information](#migration-from-lavaplayers-built-in-youtube-source)
@@ -329,6 +331,24 @@ plugins:
       userAgent: "your_service_name" # Optional user-agent header, used for metrics on the backend. 
 ```
 
+## Debugging / Saving raw responses
+
+When troubleshooting playback issues (e.g. OAuth problems, stream decoding errors), it can be helpful to capture the HTTP request and response.
+
+### Lavaplayer
+```java
+YoutubeSourceOptions options = new YoutubeSourceOptions()
+    .setDebugSaveResponsesDirectory("/path/to/debug/output");
+YoutubeAudioSourceManager sourceManager = new YoutubeAudioSourceManager(options, ...);
+```
+
+### Lavalink
+```yaml
+plugins:
+  youtube:
+    debugSaveResponsesDirectory: "/path/to/debug/output"
+```
+
 ## REST routes (`plugin` only)
 ### `POST` `/youtube`
 

common/src/main/java/dev/lavalink/youtube/YoutubeAudioSourceManager.java

@@ -3,6 +3,7 @@
 import com.sedmelluq.discord.lavaplayer.player.AudioPlayerManager;
 import com.sedmelluq.discord.lavaplayer.source.AudioSourceManager;
 import com.sedmelluq.discord.lavaplayer.tools.DataFormatTools;
+import com.sedmelluq.discord.lavaplayer.tools.io.HttpConfigurable;
 import com.sedmelluq.discord.lavaplayer.tools.ExceptionTools;
 import com.sedmelluq.discord.lavaplayer.tools.FriendlyException;
 import com.sedmelluq.discord.lavaplayer.tools.FriendlyException.Severity;
@@ -39,12 +40,17 @@
 import java.util.List;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
+import java.util.function.Consumer;
+import java.util.function.Function;
 import java.util.stream.Collectors;
 
+import org.apache.http.client.config.RequestConfig;
+import org.apache.http.impl.client.HttpClientBuilder;
+
 import static com.sedmelluq.discord.lavaplayer.tools.FriendlyException.Severity.SUSPICIOUS;
 
 @SuppressWarnings("RegExpUnnecessaryNonCapturingGroup")
-public class YoutubeAudioSourceManager implements AudioSourceManager {
+public class YoutubeAudioSourceManager implements AudioSourceManager, HttpConfigurable {
     // TODO: connect timeout = 16000ms, read timeout = 8000ms (as observed from scraped youtube config)
     // TODO: look at possibly scraping jsUrl from WEB config to save a request
     // TODO(music): scrape config? it's identical to WEB.
@@ -160,6 +166,11 @@ public YoutubeAudioSourceManager(@NotNull YoutubeSourceOptions options,
         } else {
             this.cipherManager = new LocalSignatureCipherManager();
         }
+
+        if (!DataFormatTools.isNullOrEmpty(options.getDebugSaveResponsesDirectory())) {
+            contextFilter.setDebugSaveResponsesDirectory(options.getDebugSaveResponsesDirectory());
+            log.info("Debug response saving enabled, writing to: {}", options.getDebugSaveResponsesDirectory());
+        }
     }
 
     @Override
@@ -454,6 +465,16 @@ public HttpInterface getInterface() {
         return httpInterfaceManager.getInterface();
     }
 
+    @Override
+    public void configureRequests(Function<RequestConfig, RequestConfig> configurator) {
+        httpInterfaceManager.configureRequests(configurator);
+    }
+
+    @Override
+    public void configureBuilder(Consumer<HttpClientBuilder> configurator) {
+        httpInterfaceManager.configureBuilder(configurator);
+    }
+
     @Override
     public boolean isTrackEncodable(AudioTrack track) {
         return true;

common/src/main/java/dev/lavalink/youtube/YoutubeSourceOptions.java

@@ -9,6 +9,7 @@ public class YoutubeSourceOptions {
     private String remoteCipherUrl;
     private String remoteCipherPassword;
     private String remoteCipherUserAgent;
+    private String debugSaveResponsesDirectory;
 
     public boolean isAllowSearch() {
         return allowSearch;
@@ -57,5 +58,13 @@ public String getRemoteCipherUserAgent() {
         return remoteCipherUserAgent;
     }
 
+    @Nullable
+    public String getDebugSaveResponsesDirectory() {
+        return debugSaveResponsesDirectory;
+    }
 
+    public YoutubeSourceOptions setDebugSaveResponsesDirectory(@Nullable String debugSaveResponsesDirectory) {
+        this.debugSaveResponsesDirectory = debugSaveResponsesDirectory;
+        return this;
+    }
 }

common/src/main/java/dev/lavalink/youtube/cipher/CipherManager.java

@@ -13,6 +13,7 @@
 
 import java.io.IOException;
 import java.net.URI;
+import java.nio.charset.StandardCharsets;
 import java.util.concurrent.TimeUnit;
 
 /**
@@ -41,7 +42,7 @@ default CachedPlayerScript getPlayerScript(@NotNull HttpInterface httpInterface)
             try (CloseableHttpResponse response = httpInterface.execute(new HttpGet("https://www.youtube.com/embed/"))) {
                 HttpClientTools.assertSuccessWithContent(response, "fetch player script (embed)");
 
-                String responseText = EntityUtils.toString(response.getEntity());
+                String responseText = EntityUtils.toString(response.getEntity(), StandardCharsets.UTF_8);
                 String scriptUrl = DataFormatTools.extractBetween(responseText, "\"jsUrl\":\"", "\"");
 
                 if (scriptUrl == null) {

common/src/main/java/dev/lavalink/youtube/clients/Web.java

@@ -19,6 +19,7 @@
 
 import java.io.IOException;
 import java.net.URI;
+import java.nio.charset.StandardCharsets;
 import java.net.URISyntaxException;
 import java.util.List;
 import java.util.Map;
@@ -70,7 +71,7 @@ protected void fetchClientConfig(@NotNull HttpInterface httpInterface) {
             HttpClientTools.assertSuccessWithContent(response, "client config fetch");
             lastConfigUpdate = System.currentTimeMillis();
 
-            String page = EntityUtils.toString(response.getEntity());
+            String page = EntityUtils.toString(response.getEntity(), StandardCharsets.UTF_8);
             Matcher m = CONFIG_REGEX.matcher(page);
 
             if (!m.find()) {

common/src/main/java/dev/lavalink/youtube/clients/skeleton/Client.java

@@ -46,37 +46,46 @@ public interface Client {
     default PlayabilityStatus getPlayabilityStatus(@NotNull JsonBrowser playabilityStatus,
                                                    boolean throwOnNotOk) throws CannotBeLoaded {
         String status = playabilityStatus.get("status").text();
+        String reason = playabilityStatus.get("reason").safeText();
 
         if (playabilityStatus.isNull() || status == null) {
             throw new RuntimeException("No playability status block.");
         }
 
+        org.slf4j.Logger log = org.slf4j.LoggerFactory.getLogger(Client.class);
+        log.debug("Playability status check: status={}, reason={}, throwOnNotOk={}", status, reason, throwOnNotOk);
+
         switch (status) {
             case "OK":
+                log.debug("Playability status OK");
                 return PlayabilityStatus.OK;
             case "ERROR":
-                String reason = playabilityStatus.get("reason").text();
+                String errorReason = playabilityStatus.get("reason").text();
+                log.debug("Playability status ERROR: {}", errorReason);
 
-//                if (reason.contains("This video is unavailable")) {
-//                    throw new CannotBeLoaded(new FriendlyException(reason, COMMON, null));
+//                if (errorReason.contains("This video is unavailable")) {
+//                    throw new CannotBeLoaded(new FriendlyException(errorReason, COMMON, null));
 //                }
 
-                throw new FriendlyException(reason, COMMON, null);
+                throw new FriendlyException(errorReason, COMMON, null);
             case "UNPLAYABLE":
                 String unplayableReason = getUnplayableReason(playabilityStatus);
+                log.debug("Playability status UNPLAYABLE: {}", unplayableReason);
 
                 if (unplayableReason == null) {
                     // We should have a reason so this is suspicious.
                     throw new FriendlyException("This video is unplayable.", SUSPICIOUS, null);
                 }
 
                 if (unplayableReason.contains("Playback on other websites has been disabled by the video owner") && !throwOnNotOk) {
+                    log.debug("Video is non-embeddable, returning NON_EMBEDDABLE status");
                     return PlayabilityStatus.NON_EMBEDDABLE;
                 }
 
                 throw new FriendlyException(unplayableReason, COMMON, null);
             case "LOGIN_REQUIRED":
                 String loginReason = playabilityStatus.get("reason").safeText();
+                log.debug("Playability status LOGIN_REQUIRED: {}", loginReason);
 
                 if (loginReason.contains("This video is private")) {
                     throw new CannotBeLoaded(new FriendlyException("This is a private video.", COMMON, null));

common/src/main/java/dev/lavalink/youtube/clients/skeleton/NonMusicClient.java

@@ -12,13 +12,10 @@
 import dev.lavalink.youtube.cipher.CipherManager.CachedPlayerScript;
 import dev.lavalink.youtube.clients.ClientConfig;
 import dev.lavalink.youtube.track.TemporalInfo;
-import org.apache.http.HttpResponse;
 import org.apache.http.client.methods.CloseableHttpResponse;
 import org.apache.http.client.methods.HttpGet;
 import org.apache.http.client.methods.HttpPost;
 import org.apache.http.entity.StringEntity;
-import org.apache.http.impl.client.CloseableHttpClient;
-import org.apache.http.impl.client.HttpClients;
 import org.apache.http.util.EntityUtils;
 import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
@@ -74,7 +71,7 @@ protected JsonBrowser loadJsonResponse(@NotNull HttpInterface httpInterface,
             //       from my testing, json is always returned so might not be necessary.
             HttpClientTools.assertJsonContentType(response);
 
-            String json = EntityUtils.toString(response.getEntity());
+            String json = EntityUtils.toString(response.getEntity(), StandardCharsets.UTF_8);
             log.trace("Response from {} ({}) {}", request.getURI(), context, json);
 
             return JsonBrowser.parse(json);
@@ -133,7 +130,7 @@ protected JsonBrowser loadTrackInfoFromInnertube(@NotNull YoutubeAudioSourceMana
 
         // For embedded clients, fetch and include encryptedHostFlags to avoid playback restrictions.
         if (isEmbedded()) {
-            String encryptedHostFlags = fetchEncryptedHostFlags(videoId);
+            String encryptedHostFlags = fetchEncryptedHostFlags(httpInterface, videoId);
             if (encryptedHostFlags != null) {
                 config.withEncryptedHostFlags(encryptedHostFlags);
             }
@@ -156,10 +153,19 @@ protected JsonBrowser loadTrackInfoFromInnertube(@NotNull YoutubeAudioSourceMana
         JsonBrowser playabilityJson = json.get("playabilityStatus");
         JsonBrowser videoDetails = json.get("videoDetails");
 
+        log.debug("Client {} received player API response for video {}: playabilityStatus={}, videoDetails present={}", 
+            getIdentifier(), 
+            videoId,
+            playabilityJson.isNull() ? "null" : playabilityJson.get("status").text(),
+            !videoDetails.isNull());
+
         // we should always check playabilityStatus if videoDetails is null because it could contain important
         // information as to why, which prevents false reports about this not working as intended etc etc.
         if (validatePlayabilityStatus || videoDetails.isNull()) {
             // fix: Make this method throw if a status was supplied (typically when we recurse).
+            String playabilityStatusStr = playabilityJson.isNull() ? "null" : playabilityJson.get("status").text();
+            log.debug("Validating playability status for video {} with client {}: status={}, previousStatus={}", 
+                videoId, getIdentifier(), playabilityStatusStr, status);
             PlayabilityStatus playabilityStatus = getPlayabilityStatus(playabilityJson, status != null);
 
             // All other branches should've been caught by getPlayabilityStatus().
@@ -178,44 +184,54 @@ protected JsonBrowser loadTrackInfoFromInnertube(@NotNull YoutubeAudioSourceMana
         }
 
         if (videoDetails.isNull()) {
+            log.warn("Client {} received null videoDetails for video {}. Playability status: {}, Full JSON: {}", 
+                getIdentifier(), videoId, playabilityJson.format(), json.format());
             throw new FriendlyException("Loading information for video failed", Severity.SUSPICIOUS,
                 new RuntimeException("Missing videoDetails block, JSON: " + json.format()));
         }
 
-        if (!videoId.equals(videoDetails.get("videoId").text())) {
+        String returnedVideoId = videoDetails.get("videoId").text();
+        if (!videoId.equals(returnedVideoId)) {
+            log.warn("Client {} returned incorrect video ID. Expected: {}, Got: {}, Full JSON: {}", 
+                getIdentifier(), videoId, returnedVideoId, json.format());
             throw new FriendlyException(
                 "The video returned is not what was requested.",
                 Severity.SUSPICIOUS,
                 new RuntimeException("Incorrect video response, JSON: " + json.format())
             );
         }
+
+        log.debug("Client {} successfully loaded video details for video {}", getIdentifier(), videoId);
 
         return json;
     }
 
     /**
      * Fetches the encryptedHostFlags from the YouTube embed page.
      * This is required for embedded clients to avoid playback restrictions.
+     * Uses the given HttpInterface so proxy and other HTTP config (e.g. from the player manager) are applied.
      *
+     * @param httpInterface The HTTP interface to use (respects proxy and builder configurator).
      * @param videoId The video ID to fetch the embed page for.
      * @return The encryptedHostFlags value, or null if not found.
      */
     @Nullable
-    protected String fetchEncryptedHostFlags(@NotNull String videoId) {
+    protected String fetchEncryptedHostFlags(@NotNull HttpInterface httpInterface, @NotNull String videoId) {
         String embedUrl = "https://www.youtube.com/embed/" + videoId;
 
-        try (CloseableHttpClient httpClient = HttpClients.createDefault()) {
+        try {
             HttpGet request = new HttpGet(embedUrl);
             request.setHeader("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36");
 
-            HttpResponse response = httpClient.execute(request);
-            String html = EntityUtils.toString(response.getEntity());
+            try (CloseableHttpResponse response = httpInterface.execute(request)) {
+                String html = EntityUtils.toString(response.getEntity(), StandardCharsets.UTF_8);
 
-            Pattern pattern = Pattern.compile("\"encryptedHostFlags\":\"([^\"]+)\"");
-            Matcher matcher = pattern.matcher(html);
+                Pattern pattern = Pattern.compile("\"encryptedHostFlags\":\"([^\"]+)\"");
+                Matcher matcher = pattern.matcher(html);
 
-            if (matcher.find()) {
-                return matcher.group(1);
+                if (matcher.find()) {
+                    return matcher.group(1);
+                }
             }
         } catch (IOException e) {
             log.debug("Failed to fetch encryptedHostFlags for video {}", videoId, e);

common/src/main/java/dev/lavalink/youtube/clients/skeleton/StreamingNonMusicClient.java

@@ -32,22 +32,35 @@ public abstract class StreamingNonMusicClient extends NonMusicClient {
     public TrackFormats loadFormats(@NotNull YoutubeAudioSourceManager source,
                                     @NotNull HttpInterface httpInterface,
                                     @NotNull String videoId) throws CannotBeLoaded, IOException {
+        log.debug("Client {} loading formats for video {}", getIdentifier(), videoId);
         JsonBrowser json = loadTrackInfoFromInnertube(source, httpInterface, videoId, null, true);
         JsonBrowser playabilityStatus = json.get("playabilityStatus");
         JsonBrowser videoDetails = json.get("videoDetails");
         CachedPlayerScript playerScript = source.getCipherManager().getCachedPlayerScript(httpInterface);
 
         boolean isLive = videoDetails.get("isLive").asBoolean(false);
+        log.debug("Client {} video {} isLive={}", getIdentifier(), videoId, isLive);
 
         if ("OK".equals(playabilityStatus.get("status").text()) && playabilityStatus.get("reason").safeText().contains("This live event has ended")) {
             // Long videos after ending of stream don't contain contentLength field as they
             // are still being processed by YouTube.
+            log.debug("Client {} detected ended live event for video {}", getIdentifier(), videoId);
             isLive = true;
         }
 
         JsonBrowser streamingData = json.get("streamingData");
+        if (streamingData.isNull()) {
+            log.warn("Client {} received null streamingData for video {}, full JSON: {}", 
+                getIdentifier(), videoId, json.format());
+        }
         JsonBrowser mergedFormats = streamingData.get("formats");
         JsonBrowser adaptiveFormats = streamingData.get("adaptiveFormats");
+
+        log.debug("Client {} found {} merged formats and {} adaptive formats for video {}", 
+            getIdentifier(),
+            mergedFormats.isNull() ? 0 : mergedFormats.values().size(),
+            adaptiveFormats.isNull() ? 0 : adaptiveFormats.values().size(),
+            videoId);
 
         List<StreamFormat> formats = new ArrayList<>();
         boolean anyFailures = false;
@@ -65,9 +78,12 @@ public TrackFormats loadFormats(@NotNull YoutubeAudioSourceManager source,
         }
 
         if (formats.isEmpty() && anyFailures) {
-            log.warn("Loading formats either failed to load or were skipped due to missing fields, json: {}", streamingData.format());
+            log.warn("Client {} loading formats either failed to load or were skipped due to missing fields for video {}, json: {}", 
+                getIdentifier(), videoId, streamingData.format());
         }
 
+        log.debug("Client {} successfully extracted {} formats for video {}", 
+            getIdentifier(), formats.size(), videoId);
         return new TrackFormats(formats, playerScript.url);
     }
 
@@ -86,7 +102,9 @@ protected boolean extractFormat(JsonBrowser formatJson,
             : Collections.emptyMap();
 
         if (DataFormatTools.isNullOrEmpty(url) && DataFormatTools.isNullOrEmpty(cipherInfo.get("url"))) {
-            log.debug("Client '{}' is missing format URL for itag '{}'. SABR response?", getIdentifier(), formatJson.get("itag").text());
+            String itag = formatJson.get("itag").text();
+            log.debug("Client '{}' is missing format URL for itag '{}'. SABR response? Format JSON: {}", 
+                getIdentifier(), itag, formatJson.format());
             return false;
         }