diff --git a/Cargo.lock b/Cargo.lock index 69491f5a996..0f76910744f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -836,9 +836,9 @@ dependencies = [ [[package]] name = "crates-io" -version = "0.40.20" +version = "0.41.0" dependencies = [ - "curl", + "http", "percent-encoding", "serde", "serde_json", diff --git a/Cargo.toml b/Cargo.toml index e8dcdca0a6b..88a18d84027 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -43,7 +43,7 @@ clap = "4.6.0" clap_complete = { version = "4.6.0", features = ["unstable-dynamic"] } color-print = "0.3.7" core-foundation = { version = "0.10.1", features = ["mac_os_10_7_support"] } -crates-io = { version = "0.40.20", path = "crates/crates-io" } +crates-io = { version = "0.41.0", path = "crates/crates-io" } criterion = { version = "0.8.2", features = ["html_reports"] } curl = "0.4.49" curl-sys = "0.4.87" diff --git a/crates/crates-io/Cargo.toml b/crates/crates-io/Cargo.toml index 70f1adf08eb..e189bfde807 100644 --- a/crates/crates-io/Cargo.toml +++ b/crates/crates-io/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "crates-io" -version = "0.40.20" +version = "0.41.0" rust-version = "1.95" # MSRV:1 edition.workspace = true license.workspace = true @@ -14,7 +14,7 @@ name = "crates_io" path = "lib.rs" [dependencies] -curl.workspace = true +http.workspace = true percent-encoding.workspace = true serde = { workspace = true, features = ["derive"] } serde_json.workspace = true diff --git a/crates/crates-io/lib.rs b/crates/crates-io/lib.rs index d95ce91c888..b99ad89ee07 100644 --- a/crates/crates-io/lib.rs +++ b/crates/crates-io/lib.rs @@ -7,21 +7,30 @@ use std::io::prelude::*; use std::io::{Cursor, SeekFrom}; use std::time::Instant; -use curl::easy::{Easy, List}; +use http::{Method, Request, Response, StatusCode}; use percent_encoding::{NON_ALPHANUMERIC, percent_encode}; use serde::{Deserialize, Serialize}; use url::Url; -pub type Result = std::result::Result; +type RegistryResult = Result>; -pub struct Registry { +/// Perform an HTTP request and return the response. +/// +/// Users of this crate must provide an implementation of this +/// trait using an HTTP crate such as `curl`, `reqwest`, etc. +pub trait HttpClient { + type Error: std::error::Error + Send + Sync; + fn request(&self, req: Request>) -> Result>, Self::Error>; +} + +pub struct Registry { /// The base URL for issuing API requests. host: String, /// Optional authorization token. /// If None, commands requiring authorization will fail. token: Option, - /// Curl handle for issuing requests. - handle: Easy, + /// HTTP handle for issuing requests. + handle: T, /// Whether to include the authorization token with all requests. auth_required: bool, } @@ -138,10 +147,15 @@ struct Crates { /// Error returned when interacting with a registry. #[derive(Debug, thiserror::Error)] -pub enum Error { - /// Error from libcurl. +#[non_exhaustive] +pub enum Error { + /// Error from underlying transport. + #[error(transparent)] + Transport(T), + + /// Error from http. #[error(transparent)] - Curl(#[from] curl::Error), + Http(#[from] http::Error), /// Error from serializing the request payload and deserializing the /// response body (like response body didn't match expected structure). @@ -163,25 +177,25 @@ pub enum Error { errors.join(", "), )] Api { - code: u32, + code: StatusCode, headers: Vec, errors: Vec, }, /// Error from API response which didn't have pre-programmed `errors.details`. #[error( - "failed to get a 200 OK response, got {code}\nheaders:\n\t{}\nbody:\n{body}", + "failed to get a 200 OK response, got {}\nheaders:\n\t{}\nbody:\n{body}", + code.as_u16(), headers.join("\n\t"), )] Code { - code: u32, + code: StatusCode, headers: Vec, body: String, }, - /// Reason why the token was invalid. - #[error("{0}")] - InvalidToken(&'static str), + #[error(transparent)] + InvalidToken(#[from] TokenError), /// Server was unavailable and timed out. Happened when uploading a way /// too large tarball to crates.io. @@ -194,27 +208,28 @@ pub enum Error { Timeout(u64), } -impl Registry { +impl Registry { /// Creates a new `Registry`. /// /// ## Example /// /// ```rust - /// use curl::easy::Easy; - /// use crates_io::Registry; + /// use crates_io::{Registry, HttpClient}; + /// use http::{Request, Response}; + /// + /// struct Client {} + /// impl HttpClient for Client { + /// type Error = std::io::Error; + /// fn request(&self, req: Request>) -> Result>, Self::Error> { + /// todo!() + /// } + /// } + /// let client = Client {}; /// - /// let mut handle = Easy::new(); - /// // If connecting to crates.io, a user-agent is required. - /// handle.useragent("my_crawler (example.com/info)"); - /// let mut reg = Registry::new_handle(String::from("https://crates.io"), None, handle, true); + /// let mut reg = Registry::new_handle(String::from("https://crates.io"), None, client, false); /// ``` - pub fn new_handle( - host: String, - token: Option, - handle: Easy, - auth_required: bool, - ) -> Registry { - Registry { + pub fn new_handle(host: String, token: Option, handle: T, auth_required: bool) -> Self { + Self { host, token, handle, @@ -226,10 +241,8 @@ impl Registry { self.token = token; } - fn token(&self) -> Result<&str> { - let token = self.token.as_ref().ok_or_else(|| { - Error::InvalidToken("no upload token found, please run `cargo login`") - })?; + fn token(&self) -> RegistryResult<&str, T::Error> { + let token = self.token.as_ref().ok_or_else(|| TokenError::Missing)?; check_token(token)?; Ok(token) } @@ -242,26 +255,30 @@ impl Registry { is_url_crates_io(&self.host) } - pub fn add_owners(&mut self, krate: &str, owners: &[&str]) -> Result { + pub fn add_owners(&mut self, krate: &str, owners: &[&str]) -> RegistryResult { let body = serde_json::to_string(&OwnersReq { users: owners })?; let body = self.put(&format!("/crates/{}/owners", krate), Some(body.as_bytes()))?; assert!(serde_json::from_str::(&body)?.ok); Ok(serde_json::from_str::(&body)?.msg) } - pub fn remove_owners(&mut self, krate: &str, owners: &[&str]) -> Result<()> { + pub fn remove_owners(&mut self, krate: &str, owners: &[&str]) -> RegistryResult<(), T::Error> { let body = serde_json::to_string(&OwnersReq { users: owners })?; let body = self.delete(&format!("/crates/{}/owners", krate), Some(body.as_bytes()))?; assert!(serde_json::from_str::(&body)?.ok); Ok(()) } - pub fn list_owners(&mut self, krate: &str) -> Result> { + pub fn list_owners(&mut self, krate: &str) -> RegistryResult, T::Error> { let body = self.get(&format!("/crates/{}/owners", krate))?; Ok(serde_json::from_str::(&body)?.users) } - pub fn publish(&mut self, krate: &NewCrate, mut tarball: &File) -> Result { + pub fn publish( + &mut self, + krate: &NewCrate, + mut tarball: &File, + ) -> RegistryResult { let json = serde_json::to_string(krate)?; // Prepare the body. The format of the upload request is: // @@ -284,33 +301,27 @@ impl Registry { w.extend(&(tarball_len as u32).to_le_bytes()); w }; - let size = tarball_len as usize + header.len(); - let mut body = Cursor::new(header).chain(tarball); - - let url = format!("{}/api/v1/crates/new", self.host); - - self.handle.put(true)?; - self.handle.url(&url)?; - self.handle.in_filesize(size as u64)?; - let mut headers = List::new(); - headers.append("Content-Type: application/octet-stream")?; - headers.append("Accept: application/json")?; - headers.append(&format!("Authorization: {}", self.token()?))?; - self.handle.http_headers(headers)?; - + let mut body = Vec::new(); + Cursor::new(header).chain(tarball).read_to_end(&mut body)?; + let url = self.api_url("/crates/new"); + + let request = http::Request::put(url) + .header(http::header::CONTENT_TYPE, "application/octet-stream") + .header(http::header::ACCEPT, "application/json") + .header(http::header::AUTHORIZATION, self.token()?) + .body(body)?; let started = Instant::now(); - let body = self - .handle(&mut |buf| body.read(buf).unwrap_or(0)) - .map_err(|e| match e { - Error::Code { code, .. } - if code == 503 - && started.elapsed().as_secs() >= 29 - && self.host_is_crates_io() => - { - Error::Timeout(tarball_len) - } - _ => e.into(), - })?; + let response = self.handle.request(request).map_err(Error::Transport)?; + let body = self.handle(response).map_err(|e| match e { + Error::Code { code, .. } + if code == StatusCode::SERVICE_UNAVAILABLE + && started.elapsed().as_secs() >= 29 + && self.host_is_crates_io() => + { + Error::Timeout(tarball_len) + } + _ => e.into(), + })?; let response = if body.is_empty() { "{}".parse()? @@ -346,9 +357,14 @@ impl Registry { }) } - pub fn search(&mut self, query: &str, limit: u32) -> Result<(Vec, u32)> { + pub fn search( + &mut self, + query: &str, + limit: u32, + ) -> RegistryResult<(Vec, u32), T::Error> { let formatted_query = percent_encode(query.as_bytes(), NON_ALPHANUMERIC); let body = self.req( + Method::GET, &format!("/crates?q={}&per_page={}", formatted_query, limit), None, Auth::Unauthorized, @@ -358,88 +374,81 @@ impl Registry { Ok((crates.crates, crates.meta.total)) } - pub fn yank(&mut self, krate: &str, version: &str) -> Result<()> { + pub fn yank(&mut self, krate: &str, version: &str) -> RegistryResult<(), T::Error> { let body = self.delete(&format!("/crates/{}/{}/yank", krate, version), None)?; assert!(serde_json::from_str::(&body)?.ok); Ok(()) } - pub fn unyank(&mut self, krate: &str, version: &str) -> Result<()> { + pub fn unyank(&mut self, krate: &str, version: &str) -> RegistryResult<(), T::Error> { let body = self.put(&format!("/crates/{}/{}/unyank", krate, version), None)?; assert!(serde_json::from_str::(&body)?.ok); Ok(()) } - fn put(&mut self, path: &str, b: Option<&[u8]>) -> Result { - self.handle.put(true)?; - self.req(path, b, Auth::Authorized) + fn put(&mut self, path: &str, b: Option<&[u8]>) -> RegistryResult { + self.req(Method::PUT, path, b, Auth::Authorized) } - fn get(&mut self, path: &str) -> Result { - self.handle.get(true)?; - self.req(path, None, Auth::Authorized) + fn get(&mut self, path: &str) -> RegistryResult { + self.req(Method::GET, path, None, Auth::Authorized) } - fn delete(&mut self, path: &str, b: Option<&[u8]>) -> Result { - self.handle.custom_request("DELETE")?; - self.req(path, b, Auth::Authorized) + fn delete(&mut self, path: &str, b: Option<&[u8]>) -> RegistryResult { + self.req(Method::DELETE, path, b, Auth::Authorized) + } + + fn api_url(&self, path: &str) -> String { + // http::Uri doesn't support file urls without an authority, even though it's optional. + // We insert localhost here to make it work. + let host = &self.host; + if let Some(file_url) = host.strip_prefix("file:///") { + format!("file://localhost/{file_url}/api/v1{path}") + } else { + format!("{host}/api/v1{path}") + } } - fn req(&mut self, path: &str, body: Option<&[u8]>, authorized: Auth) -> Result { - self.handle.url(&format!("{}/api/v1{}", self.host, path))?; - let mut headers = List::new(); - headers.append("Accept: application/json")?; + fn req( + &mut self, + method: Method, + path: &str, + body: Option<&[u8]>, + authorized: Auth, + ) -> RegistryResult { + let url = self.api_url(path); + let mut request = http::Request::builder() + .method(method) + .uri(url) + .header(http::header::ACCEPT, "application/json"); if body.is_some() { - headers.append("Content-Type: application/json")?; + request = request.header(http::header::CONTENT_TYPE, "Content-Type: application/json"); } if self.auth_required || authorized == Auth::Authorized { - headers.append(&format!("Authorization: {}", self.token()?))?; - } - self.handle.http_headers(headers)?; - match body { - Some(mut body) => { - self.handle.upload(true)?; - self.handle.in_filesize(body.len() as u64)?; - self.handle(&mut |buf| body.read(buf).unwrap_or(0)) - .map_err(|e| e.into()) - } - None => self.handle(&mut |_| 0).map_err(|e| e.into()), + request = request.header(http::header::AUTHORIZATION, self.token()?); } + let request = request.body(body.unwrap_or_default().to_vec())?; + let response = self.handle.request(request).map_err(Error::Transport)?; + self.handle(response) } - fn handle(&mut self, read: &mut dyn FnMut(&mut [u8]) -> usize) -> Result { - let mut headers = Vec::new(); - let mut body = Vec::new(); - { - let mut handle = self.handle.transfer(); - handle.read_function(|buf| Ok(read(buf)))?; - handle.write_function(|data| { - body.extend_from_slice(data); - Ok(data.len()) - })?; - handle.header_function(|data| { - // Headers contain trailing \r\n, trim them to make it easier - // to work with. - let s = String::from_utf8_lossy(data).trim().to_string(); - // Don't let server sneak extra lines anywhere. - if s.contains('\n') { - return true; - } - headers.push(s); - true - })?; - handle.perform()?; - } - + fn handle(&mut self, response: http::Response>) -> RegistryResult { + let (head, body) = response.into_parts(); let body = String::from_utf8(body)?; let errors = serde_json::from_str::(&body) .ok() .map(|s| s.errors.into_iter().map(|s| s.detail).collect::>()); - match (self.handle.response_code()?, errors) { - (0, None) => Ok(body), - (code, None) if is_success(code) => Ok(body), + let headers = head + .headers + .iter() + .filter_map(|(k, v)| Some((k, v.to_str().ok()?))) + .map(|(k, v)| format!("{k}: {v}")) + .collect(); + + match (head.status, errors) { + (code, None) if code.is_success() => Ok(body), (code, Some(errors)) => Err(Error::Api { code, headers, @@ -454,65 +463,11 @@ impl Registry { } } -fn is_success(code: u32) -> bool { - code >= 200 && code < 300 -} - -fn status(code: u32) -> String { - if is_success(code) { +fn status(code: StatusCode) -> String { + if code.is_success() { String::new() } else { - let reason = reason(code); - format!(" (status {code} {reason})") - } -} - -fn reason(code: u32) -> &'static str { - // Taken from https://developer.mozilla.org/en-US/docs/Web/HTTP/Status - match code { - 100 => "Continue", - 101 => "Switching Protocol", - 103 => "Early Hints", - 200 => "OK", - 201 => "Created", - 202 => "Accepted", - 203 => "Non-Authoritative Information", - 204 => "No Content", - 205 => "Reset Content", - 206 => "Partial Content", - 300 => "Multiple Choice", - 301 => "Moved Permanently", - 302 => "Found", - 303 => "See Other", - 304 => "Not Modified", - 307 => "Temporary Redirect", - 308 => "Permanent Redirect", - 400 => "Bad Request", - 401 => "Unauthorized", - 402 => "Payment Required", - 403 => "Forbidden", - 404 => "Not Found", - 405 => "Method Not Allowed", - 406 => "Not Acceptable", - 407 => "Proxy Authentication Required", - 408 => "Request Timeout", - 409 => "Conflict", - 410 => "Gone", - 411 => "Length Required", - 412 => "Precondition Failed", - 413 => "Payload Too Large", - 414 => "URI Too Long", - 415 => "Unsupported Media Type", - 416 => "Request Range Not Satisfiable", - 417 => "Expectation Failed", - 429 => "Too Many Requests", - 431 => "Request Header Fields Too Large", - 500 => "Internal Server Error", - 501 => "Not Implemented", - 502 => "Bad Gateway", - 503 => "Service Unavailable", - 504 => "Gateway Timeout", - _ => "", + format!(" (status {code})") } } @@ -523,14 +478,29 @@ pub fn is_url_crates_io(url: &str) -> bool { .unwrap_or(false) } +#[derive(Debug, thiserror::Error)] +pub enum TokenError { + #[error("no upload token found, please run `cargo login`")] + Missing, + + #[error("please provide a non-empty token")] + Empty, + + #[error( + "token contains invalid characters.\nOnly printable ISO-8859-1 characters \ + are allowed as it is sent in a HTTPS header." + )] + InvalidCharacters, +} + /// Checks if a token is valid or malformed. /// /// This check is necessary to prevent sending tokens which create an invalid HTTP request. /// It would be easier to check just for alphanumeric tokens, but we can't be sure that all /// registries only create tokens in that format so that is as less restricted as possible. -pub fn check_token(token: &str) -> Result<()> { +pub fn check_token(token: &str) -> Result<(), TokenError> { if token.is_empty() { - return Err(Error::InvalidToken("please provide a non-empty token")); + return Err(TokenError::Empty); } if token.bytes().all(|b| { // This is essentially the US-ASCII limitation of @@ -541,9 +511,6 @@ pub fn check_token(token: &str) -> Result<()> { }) { Ok(()) } else { - Err(Error::InvalidToken( - "token contains invalid characters.\nOnly printable ISO-8859-1 characters \ - are allowed as it is sent in a HTTPS header.", - )) + Err(TokenError::InvalidCharacters) } } diff --git a/src/cargo/ops/registry/mod.rs b/src/cargo/ops/registry/mod.rs index da4cd2edc3a..62320da2527 100644 --- a/src/cargo/ops/registry/mod.rs +++ b/src/cargo/ops/registry/mod.rs @@ -25,7 +25,7 @@ use crate::util::auth; use crate::util::cache_lock::CacheLockMode; use crate::util::context::{GlobalContext, PathAndArgs}; use crate::util::errors::CargoResult; -use crate::util::network::http::http_handle; +use crate::util::network::http_async; pub use self::info::info; pub use self::login::registry_login; @@ -126,7 +126,7 @@ fn registry<'gctx>( reg_or_index: Option<&RegistryOrIndex>, force_update: bool, token_required: Option>, -) -> CargoResult<(Registry, RegistrySource<'gctx>)> { +) -> CargoResult<(Registry>, RegistrySource<'gctx>)> { let is_index = reg_or_index.map(|v| v.is_index()).unwrap_or_default(); if is_index && token_required.is_some() && token_from_cmdline.is_none() { bail!("command-line argument --index requires --token to be specified"); @@ -162,7 +162,7 @@ fn registry<'gctx>( } else { None }; - let handle = http_handle(gctx)?; + let handle = RegistryClient(gctx.http_async()?); Ok(( Registry::new_handle(api_host, token, handle, cfg.auth_required), src, @@ -362,3 +362,13 @@ pub(crate) fn infer_registry(pkgs: &[&Package]) -> CargoResult(&'gctx http_async::Client); + +impl<'gctx> crates_io::HttpClient for RegistryClient<'gctx> { + type Error = http_async::Error; + + fn request(&self, req: http::Request>) -> Result>, Self::Error> { + self.0.request_blocking(req) + } +} diff --git a/src/cargo/ops/registry/publish.rs b/src/cargo/ops/registry/publish.rs index f90744341e5..48908d02e73 100644 --- a/src/cargo/ops/registry/publish.rs +++ b/src/cargo/ops/registry/publish.rs @@ -37,6 +37,7 @@ use crate::ops; use crate::ops::PackageOpts; use crate::ops::Packages; use crate::ops::RegistryOrIndex; +use crate::ops::registry::RegistryClient; use crate::ops::registry::RegistrySourceIds; use crate::sources::CRATES_IO_REGISTRY; use crate::sources::RegistrySource; @@ -487,7 +488,7 @@ fn verify_unpublished( fn verify_dependencies( pkg: &Package, - registry: &Registry, + registry: &Registry>, registry_src: SourceId, ) -> CargoResult<()> { for dep in pkg.dependencies().iter() { @@ -652,7 +653,7 @@ fn transmit( ws: &Workspace<'_>, pkg: &Package, tarball: &File, - registry: &mut Registry, + registry: &mut Registry>, registry_id: SourceId, dry_run: bool, workspace_context: impl Fn() -> String, diff --git a/src/cargo/util/network/http_async.rs b/src/cargo/util/network/http_async.rs index 25b371269e5..9f1343ccadb 100644 --- a/src/cargo/util/network/http_async.rs +++ b/src/cargo/util/network/http_async.rs @@ -98,7 +98,10 @@ impl Client { /// Perform a blocking HTTP request using this client. /// Does not start an async executor. pub fn request_blocking(&self, request: Request) -> HttpResult { - let handle = self.request_helper(request)?; + let mut handle = self.request_helper(request)?; + // Configure the handle timeout since we're blocking here and not using the + // client-level timeout. + self.handle_config.timeout.configure2(&mut handle)?; handle.perform()?; Ok(WorkerServer::process_response(handle)) } @@ -142,8 +145,10 @@ impl Client { handle.put(true)?; } method => { - handle.upload(true)?; - handle.in_filesize(body_len as u64)?; + if body_len > 0 { + handle.upload(true)?; + handle.in_filesize(body_len as u64)?; + } handle.custom_request(method.as_str())?; } } diff --git a/tests/testsuite/publish.rs b/tests/testsuite/publish.rs index 2207186872a..6145b3997fe 100644 --- a/tests/testsuite/publish.rs +++ b/tests/testsuite/publish.rs @@ -2420,10 +2420,8 @@ fn api_error_code() { Caused by: failed to get a 200 OK response, got 400 headers: - HTTP/1.1 400 - Content-Length: 7 - Connection: close - + content-length: 7 + connection: close body: go away @@ -4584,11 +4582,9 @@ fn workspace_publish_rate_limit_error() { Caused by: failed to get a 200 OK response, got 429 headers: - HTTP/1.1 429 - Content-Length: 172 - Connection: close - Retry-After: 3600 - + content-length: 172 + connection: close + retry-after: 3600 body: You have published too many new crates in a short period of time. Please try again after Fri, 18 Jul 2025 20:00:34 GMT or email help@crates.io to have your limit increased.