fix(rutracker): make the Cloudflare path actually work

monitorrent/plugins/trackers/rutracker.py

@@ -15,6 +15,34 @@
 
 PLUGIN_NAME = 'rutracker.org'
 
+# Cloudflare lets a request through when it has cf_clearance plus the
+# User-Agent that cookie was issued for. Both fields take a single value copied
+# from a browser, or a json object like lostfilm's.
+
+
+def _parse_json_or_value(value, key):
+    if not value:
+        return None
+    value = value.strip()
+    if not value:
+        return None
+    if value.startswith('{'):
+        return json.loads(value)
+    return {key: value}
+
+
+def parse_cookies_field(value):
+    """bare cf_clearance value, or a json object of cookies"""
+    return _parse_json_or_value(value, 'cf_clearance')
+
+
+def parse_headers_field(value):
+    """bare User-Agent, or a json object of headers
+
+    no default on purpose: a made up agent would not match cf_clearance
+    """
+    return _parse_json_or_value(value, 'User-Agent')
+
 
 class RutrackerCredentials(Base):
     __tablename__ = "rutracker_credentials"
@@ -109,7 +137,9 @@ def parse_url(self, url):
     def login(self, username, password, headers=None, cookies=None):
         self.headers = headers
         self.cookies = cookies
-        update_headers_and_cookies_mixin(self, "https://rutracker.org/forum/index.php")
+        # probe login.php, not the index: the index is not behind the challenge,
+        # so probing it always reports "no protection" and never solves anything
+        headers, cookies = update_headers_and_cookies_mixin(self, self.login_url)
 
         username_q = username.encode('windows-1251')
         password_q = password.encode('windows-1251')
@@ -122,6 +152,11 @@ def login(self, username, password, headers=None, cookies=None):
 
         login_result = s.post(self.login_url, data, headers=headers, cookies=cookies, **kwargs)
 
+        # the challenge is served from login.php itself, so the url check below
+        # would read it as a returned login form and blame the password
+        if login_result.status_code == 403:
+            raise RutrackerLoginFailedException(3, "Blocked by Cloudflare challenge, not a credentials problem")
+
         if login_result.url.startswith(self.login_url):
             # TODO get error info (although it shouldn't contain anything useful
             # it can contain request to enter capture, so we should handle it
@@ -140,14 +175,19 @@ def verify(self):
         cookies = self.get_cookies()
         if not cookies:
             return False
-        profile_page_result = requests.get(self.profile_page, cookies=cookies,
+        profile_page_result = requests.get(self.profile_page, cookies=cookies, headers=self.headers,
                                            **self.tracker_settings.get_requests_kwargs())
-        return profile_page_result.url == self.profile_page
+        # the challenge answers 403 without redirecting, so the url alone says
+        # nothing about the session
+        return profile_page_result.status_code == 200 and profile_page_result.url == self.profile_page
 
     def get_cookies(self):
         if not self.bb_data:
             return False
-        return {'bb_session': self.bb_data}
+        # cf_clearance has to travel with every request, not just the login
+        cookies = dict(self.cookies or {})
+        cookies['bb_session'] = self.bb_data
+        return cookies
 
     def get_id(self, url):
         match = self._regex.match(url)
@@ -169,6 +209,41 @@ class RutrackerPlugin(WithCredentialsMixin, ExecuteWithHashChangeMixin, TrackerP
     tracker = RutrackerTracker()
     topic_class = RutrackerTopic
     credentials_class = RutrackerCredentials
+    # whitelists: without cookies and headers the form fields below are dropped
+    credentials_public_fields = ['username', 'cookies', 'headers']
+    credentials_private_fields = ['username', 'password', 'cookies', 'headers']
+    # same escape hatch lostfilm has: the bundled solver cannot pass the current
+    # challenge, so a cf_clearance from a real browser is what makes this work
+    credentials_form = [{
+        'type': 'row',
+        'content': [{
+            'type': 'text',
+            'model': 'username',
+            'label': 'Username',
+            'flex': 50
+        }, {
+            "type": "password",
+            "model": "password",
+            "label": "Password",
+            "flex": 50
+        }]
+    }, {
+        'type': 'row',
+        'content': [{
+            'type': 'text',
+            'model': 'cookies',
+            'label': 'cf_clearance cookie (DevTools → Application → Cookies)',
+            'flex': 100,
+        }],
+    }, {
+        'type': 'row',
+        'content': [{
+            'type': 'text',
+            'model': 'headers',
+            'label': 'User-Agent of the same browser (navigator.userAgent)',
+            'flex': 100,
+        }],
+    }]
     topic_form = [{
         'type': 'row',
         'content': [{
@@ -186,10 +261,14 @@ def login(self):
                 return LoginResult.CredentialsNotSpecified
             username = cred.username
             password = cred.password
-            headers = json.loads(cred.headers) if cred.headers else None
-            cookies = json.loads(cred.cookies) if cred.cookies else None
+            headers = parse_headers_field(cred.headers)
+            cookies = parse_cookies_field(cred.cookies)
             if not username or not password:
                 return LoginResult.CredentialsNotSpecified
+            # cf_clearance is refused with any other User-Agent, so it is the
+            # pair or nothing
+            if cookies and 'cf_clearance' in cookies and not (headers or {}).get('User-Agent'):
+                return LoginResult.CredentialsNotSpecified
         try:
             self.tracker.login(username, password, headers, cookies)
             with DBSession() as db:
@@ -216,7 +295,11 @@ def verify(self):
             password = cred.password
             if not username or not password or not cred.uid or not cred.bb_data:
                 return False
-            self.tracker.setup(cred.uid, cred.bb_data)
+            # restore cookies and headers too: login() reads them from the db
+            # itself, so without this everything after login goes out bare
+            self.tracker.setup(cred.uid, cred.bb_data,
+                               headers=parse_headers_field(cred.headers),
+                               cookies=parse_cookies_field(cred.cookies))
         return self.tracker.verify()
 
     def can_parse_url(self, url):
@@ -226,7 +309,9 @@ def parse_url(self, url):
         return self.tracker.parse_url(url)
 
     def _prepare_request(self, topic):
-        headers = {'referer': topic.url, 'host': "rutracker.org"}
+        # the cookie needs its own User-Agent; request headers still win
+        headers = dict(self.tracker.headers or {})
+        headers.update({'referer': topic.url, 'host': "rutracker.org"})
         cookies = self.tracker.get_cookies()
         request = requests.Request('POST', self.tracker.get_download_url(topic.url), headers=headers, cookies=cookies)
         return request.prepare()

tests/cassettes/test_rutrackerplugin.RutrackerPluginTest.test_login_verify_fail

@@ -12,7 +12,7 @@ interactions:
       - Mozilla/5.0 (Linux; Android 5.1.1; vivo V3 Build/LMY47V) AppleWebKit/537.36
         (KHTML, like Gecko) Chrome/54.0.2840.85 Mobile Safari/537.36
     method: GET
-    uri: https://rutracker.org/forum/index.php
+    uri: https://rutracker.org/forum/login.php
   response:
     body:
       string: !!binary |

tests/cassettes/test_rutrackerplugin.RutrackerPluginTest.test_login_verify_success

@@ -11,7 +11,7 @@ interactions:
       User-Agent:
       - Mozilla/5.0 (Android 4.4.4; Tablet; rv:54.0) Gecko/54.0 Firefox/54.0
     method: GET
-    uri: https://rutracker.org/forum/index.php
+    uri: https://rutracker.org/forum/login.php
   response:
     body:
       string: !!binary |

tests/cassettes/test_rutrackertracker.RutrackerTrackerTest.test_login

@@ -11,7 +11,7 @@ interactions:
       User-Agent:
       - Mozilla/5.0 (Macintosh; Intel Mac OS X 10.11; rv:56.0) Gecko/20100101 Firefox/56.0
     method: GET
-    uri: https://rutracker.org/forum/index.php
+    uri: https://rutracker.org/forum/login.php
   response:
     body:
       string: !!binary |

tests/cassettes/test_rutrackertracker.RutrackerTrackerTest.test_login_failed

@@ -12,7 +12,7 @@ interactions:
       - Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko)
         Chrome/70.0.3538.67 Safari/537.36
     method: GET
-    uri: https://rutracker.org/forum/index.php
+    uri: https://rutracker.org/forum/login.php
   response:
     body:
       string: !!binary |

tests/cassettes/test_rutrackertracker.RutrackerTrackerTest.test_verify

@@ -11,7 +11,7 @@ interactions:
       User-Agent:
       - Mozilla/5.0 (Windows NT 6.1; rv:53.0) Gecko/20100101 Firefox/53.0
     method: GET
-    uri: https://rutracker.org/forum/index.php
+    uri: https://rutracker.org/forum/login.php
   response:
     body:
       string: !!binary |

tests/plugins/trackers/rutracker/test_rutrackerplugin.py

@@ -2,6 +2,7 @@
 from mock import patch
 from monitorrent.plugins.trackers import LoginResult, TrackerSettings, CloudflareChallengeSolverSettings
 from monitorrent.plugins.trackers.rutracker import RutrackerPlugin, RutrackerLoginFailedException, RutrackerTopic
+from monitorrent.db import DBSession
 from tests import use_vcr, DbTestCase
 from tests.plugins.trackers import TrackerSettingsMock
 from tests.plugins.trackers.rutracker.rutracker_helper import RutrackerHelper
@@ -93,3 +94,85 @@ def test_prepare_request(self):
             self.assertEqual(request.headers['referer'], url)
             self.assertEqual(request.headers['host'], 'rutracker.org')
             self.assertEqual(request.url, 'https://rutracker.org/forum/dl.php?t=5062041')
+
+    def test_prepare_request_sends_cloudflare_headers(self):
+        # cf_clearance is issued for one User-Agent and is worthless without it
+        self.plugin.tracker.setup(self.helper.fake_uid, self.helper.fake_bb_data,
+                                  headers={'User-Agent': 'test-agent'},
+                                  cookies={'cf_clearance': 'test-clearance'})
+
+        url = 'http://rutracker.org/forum/viewtopic.php?t=5062041'
+        request = self.plugin._prepare_request(RutrackerTopic(url=url))
+
+        self.assertEqual(request.headers['User-Agent'], 'test-agent')
+        # request-specific headers must still win over the stored ones
+        self.assertEqual(request.headers['referer'], url)
+        self.assertEqual(request.headers['host'], 'rutracker.org')
+
+    @patch('monitorrent.plugins.trackers.rutracker.RutrackerPlugin.login')
+    def test_update_credentials_stores_cloudflare_cookies(self, login):
+        # cookies and headers must survive a round trip through the credentials API
+        login.return_value = LoginResult.Ok
+
+        self.plugin.update_credentials({
+            'username': self.helper.fake_login,
+            'password': self.helper.fake_password,
+            'cookies': '{"cf_clearance": "test-clearance"}',
+            'headers': '{"User-Agent": "test-agent"}',
+        })
+
+        credentials = self.plugin.get_credentials()
+        self.assertEqual(credentials['cookies'], '{"cf_clearance": "test-clearance"}')
+        self.assertEqual(credentials['headers'], '{"User-Agent": "test-agent"}')
+
+    def test_credentials_accept_bare_cookie_value(self):
+        # the one cookie worth pasting is cf_clearance — that is what Cloudflare checks
+        from monitorrent.plugins.trackers.rutracker import parse_cookies_field
+
+        self.assertEqual(parse_cookies_field('abc123'), {'cf_clearance': 'abc123'})
+        self.assertEqual(parse_cookies_field('  abc123  '), {'cf_clearance': 'abc123'})
+        self.assertEqual(parse_cookies_field('{"cf_clearance": "abc123"}'), {'cf_clearance': 'abc123'})
+        self.assertIsNone(parse_cookies_field(''))
+        self.assertIsNone(parse_cookies_field(None))
+
+    def test_credentials_accept_bare_user_agent(self):
+        # same shape for the User-Agent, and no invented default
+        from monitorrent.plugins.trackers.rutracker import parse_headers_field
+
+        self.assertEqual(parse_headers_field('My Browser 1.0'), {'User-Agent': 'My Browser 1.0'})
+        self.assertEqual(parse_headers_field('{"User-Agent": "My Browser 1.0"}'), {'User-Agent': 'My Browser 1.0'})
+        self.assertIsNone(parse_headers_field(''))
+        self.assertIsNone(parse_headers_field(None))
+
+    @patch('monitorrent.plugins.trackers.rutracker.RutrackerTracker.verify')
+    def test_verify_restores_stored_cookies_and_headers(self, tracker_verify):
+        # everything after login runs on the credentials restored here
+        tracker_verify.return_value = True
+        self.plugin.update_credentials({
+            'username': self.helper.fake_login,
+            'password': self.helper.fake_password,
+            'cookies': '{"cf_clearance": "test-clearance"}',
+            'headers': 'test-agent',
+        })
+        with DBSession() as db:
+            cred = db.query(self.plugin.credentials_class).first()
+            cred.uid = self.helper.fake_uid
+            cred.bb_data = self.helper.fake_bb_data
+
+        self.plugin.verify()
+
+        self.assertEqual(self.plugin.tracker.cookies.get('cf_clearance'), 'test-clearance')
+        self.assertEqual(self.plugin.tracker.headers.get('User-Agent'), 'test-agent')
+
+    @patch('monitorrent.plugins.trackers.rutracker.RutrackerTracker.login')
+    def test_login_requires_user_agent_with_clearance(self, tracker_login):
+        # cf_clearance without its User-Agent is refused by Cloudflare anyway
+        result = self.plugin.update_credentials({
+            'username': self.helper.fake_login,
+            'password': self.helper.fake_password,
+            'cookies': 'test-clearance',
+            'headers': '',
+        })
+
+        self.assertEqual(result, LoginResult.CredentialsNotSpecified)
+        self.assertFalse(tracker_login.called)