diff --git a/CHANGELOG.md b/CHANGELOG.md index 34b24c9766..f19e7662f4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,29 +1,3 @@ -## [v0.5.3](https://github.com/ScoopInstaller/Scoop/compare/v0.5.2...v0.5.3) - 2025-08-11 - -### Features - -**autoupdate:** GitHub predefined hashes support ([#6416](https://github.com/ScoopInstaller/Scoop/issues/6416)) - -### Bug Fixes - -- **scoop-download|install|update:** Fallback to default downloader when aria2 fails ([#4292](https://github.com/ScoopInstaller/Scoop/issues/4292)) -- **decompress**: `Expand-7zipArchive` only delete temp dir / `$extractDir` if it is empty ([#6092](https://github.com/ScoopInstaller/Scoop/issues/6092)) -- **decompress**: Replace deprecated 7ZIPEXTRACT_USE_EXTERNAL config with USE_EXTERNAL_7ZIP ([#6327](https://github.com/ScoopInstaller/Scoop/issues/6327)) -- **commands**: Handling broken aliases ([#6141](https://github.com/ScoopInstaller/Scoop/issues/6141)) -- **shim:** Do not suppress `stderr`, properly check `wslpath`/`cygpath` command first ([#6114](https://github.com/ScoopInstaller/Scoop/issues/6114)) -- **scoop-bucket:** Add missing import for `no_junction` envs ([#6181](https://github.com/ScoopInstaller/Scoop/issues/6181)) -- **scoop-uninstall:** Fix uninstaller does not gain Global state ([#6430](https://github.com/ScoopInstaller/Scoop/issues/6430)) -- **scoop-depends-tests:** Mocking `USE_EXTERNAL_7ZIP` as $false to avoding error when it is $true ([#6431](https://github.com/ScoopInstaller/Scoop/issues/6431)) - -### Code Refactoring - -- **download:** Move download-related functions to 'download.ps1' ([#6095](https://github.com/ScoopInstaller/Scoop/issues/6095)) -- **Get-Manifest:** Select actual source for manifest ([#6142](https://github.com/ScoopInstaller/Scoop/issues/6142)) - -### Performance Improvements - -- **shim:** Update kiennq-shim to v3.1.2 ([#6261](https://github.com/ScoopInstaller/Scoop/issues/6261)) - ## [v0.5.2](https://github.com/ScoopInstaller/Scoop/compare/v0.5.1...v0.5.2) - 2024-07-26 ### Bug Fixes @@ -66,7 +40,7 @@ - **checkver:** Correct error messages ([#6024](https://github.com/ScoopInstaller/Scoop/issues/6024)) - **core:** Search for Git executable instead of any cmdlet ([#5998](https://github.com/ScoopInstaller/Scoop/issues/5998)) - **core:** Use correct path in 'bash' ([#6006](https://github.com/ScoopInstaller/Scoop/issues/6006)) -- **core:** Limit the number of commands to get when search for git executable ([#6013](https://github.com/ScoopInstaller/Scoop/issues/6013)) +- **core:** Limit the number of commands to get when search for git executable ([#6013](https://github.com/ScoopInstaller/Scoop/pull/6013)) - **decompress:** Match `extract_dir`/`extract_to` and archives ([#5983](https://github.com/ScoopInstaller/Scoop/issues/5983)) - **json:** Serialize jsonpath return ([#5921](https://github.com/ScoopInstaller/Scoop/issues/5921)) - **shim:** Restore original path for JAR cmd ([#6030](https://github.com/ScoopInstaller/Scoop/issues/6030)) diff --git a/bin/checkhashes.ps1 b/bin/checkhashes.ps1 index 6e6420fb2a..cbfb0af40f 100644 --- a/bin/checkhashes.ps1 +++ b/bin/checkhashes.ps1 @@ -46,7 +46,7 @@ param( . "$PSScriptRoot\..\lib\autoupdate.ps1" . "$PSScriptRoot\..\lib\json.ps1" . "$PSScriptRoot\..\lib\versions.ps1" -. "$PSScriptRoot\..\lib\download.ps1" +. "$PSScriptRoot\..\lib\install.ps1" $Dir = Convert-Path $Dir if ($ForceUpdate) { $Update = $true } diff --git a/bin/checkurls.ps1 b/bin/checkurls.ps1 index 0ac593362a..64ab0c1840 100644 --- a/bin/checkurls.ps1 +++ b/bin/checkurls.ps1 @@ -28,7 +28,7 @@ param( . "$PSScriptRoot\..\lib\core.ps1" . "$PSScriptRoot\..\lib\manifest.ps1" -. "$PSScriptRoot\..\lib\download.ps1" +. "$PSScriptRoot\..\lib\install.ps1" $Dir = Convert-Path $Dir $Queue = @() diff --git a/bin/checkver.ps1 b/bin/checkver.ps1 index 33a4449488..57010c9992 100644 --- a/bin/checkver.ps1 +++ b/bin/checkver.ps1 @@ -73,7 +73,7 @@ param( . "$PSScriptRoot\..\lib\buckets.ps1" . "$PSScriptRoot\..\lib\json.ps1" . "$PSScriptRoot\..\lib\versions.ps1" -. "$PSScriptRoot\..\lib\download.ps1" +. "$PSScriptRoot\..\lib\install.ps1" # needed for hash generation if ($App -ne '*' -and (Test-Path $App -PathType Leaf)) { $Dir = Split-Path $App diff --git a/bin/describe.ps1 b/bin/describe.ps1 index 5faa1c403c..f9e024e4c4 100644 --- a/bin/describe.ps1 +++ b/bin/describe.ps1 @@ -23,7 +23,6 @@ param( . "$PSScriptRoot\..\lib\core.ps1" . "$PSScriptRoot\..\lib\manifest.ps1" . "$PSScriptRoot\..\lib\description.ps1" -. "$PSScriptRoot\..\lib\download.ps1" $Dir = Convert-Path $Dir $Queue = @() diff --git a/lib/autoupdate.ps1 b/lib/autoupdate.ps1 index 71c9d5b61f..bd24e04015 100644 --- a/lib/autoupdate.ps1 +++ b/lib/autoupdate.ps1 @@ -1,22 +1,4 @@ # Must included with 'json.ps1' - -function format_hash([String] $hash) { - $hash = $hash.toLower() - - if ($hash -like 'sha256:*') { - $hash = $hash.Substring(7) # Remove prefix 'sha256:' - } - - switch ($hash.Length) { - 32 { $hash = "md5:$hash" } # md5 - 40 { $hash = "sha1:$hash" } # sha1 - 64 { $hash = $hash } # sha256 - 128 { $hash = "sha512:$hash" } # sha512 - default { $hash = $null } - } - return $hash -} - function find_hash_in_rdf([String] $url, [String] $basename) { $xml = $null try { @@ -264,10 +246,6 @@ function get_hash_for_app([String] $app, $config, [String] $version, [String] $u $hashmode = 'sourceforge' } - if ($url -match 'https:\/\/github\.com\/(?[^\/]+)\/(?[^\/]+)\/releases\/download\/[^\/]+\/[^\/]+') { - $hashmode = 'github' - } - switch ($hashmode) { 'extract' { $hash = find_hash_in_textfile $hashfile_url $substitutions $regex @@ -295,10 +273,6 @@ function get_hash_for_app([String] $app, $config, [String] $version, [String] $u $hashfile_url = (strip_filename (strip_fragment "https://sourceforge.net/projects/$($matches['project'])/files/$($matches['file'])")).TrimEnd('/') $hash = find_hash_in_textfile $hashfile_url $substitutions '"$basename":.*?"sha1":\s*"([a-fA-F0-9]{40})"' } - 'github' { - $hashfile_url = "https://api.github.com/repos/$($matches['owner'])/$($matches['repo'])/releases" - $hash = find_hash_in_json $hashfile_url $substitutions ("$..assets[?(@.browser_download_url == '" + $url + "')].digest") - } } if ($hash) { diff --git a/lib/commands.ps1 b/lib/commands.ps1 index 6812aebd52..04775de357 100644 --- a/lib/commands.ps1 +++ b/lib/commands.ps1 @@ -4,7 +4,7 @@ function command_files { (Get-ChildItem "$PSScriptRoot\..\libexec") + (Get-ChildItem "$scoopdir\shims") | - Where-Object 'scoop-.*?\.ps1$' -Property Name -Match + Where-Object 'scoop-.*?\.ps1$' -Property Name -Match } function commands { @@ -86,9 +86,7 @@ function rm_alias { } info "Removing alias '$name'..." - if (Test-Path "$(shimdir $false)\scoop-$name.ps1") { - Remove-Item "$(shimdir $false)\scoop-$name.ps1" - } + Remove-Item "$(shimdir $false)\scoop-$name.ps1" $aliases.PSObject.Properties.Remove($name) set_config ALIAS $aliases | Out-Null } @@ -100,19 +98,11 @@ function list_aliases { $aliases = get_config ALIAS ([PSCustomObject]@{}) $alias_info = $aliases.PSObject.Properties.Name | Where-Object { $_ } | ForEach-Object { - # Mark the alias as , if the alias script file does NOT exist. - if (!(Test-Path "$(shimdir $false)\scoop-$_.ps1")) { - [PSCustomObject]@{ - Name = $_ - Command = '' - } - return - } $content = Get-Content (command_path $_) [PSCustomObject]@{ Name = $_ - Command = ($content | Select-Object -Skip 1).Trim() Summary = (summary $content).Trim() + Command = ($content | Select-Object -Skip 1).Trim() } } if (!$alias_info) { diff --git a/lib/core.ps1 b/lib/core.ps1 index 7ca7d121e2..23a10c32ed 100644 --- a/lib/core.ps1 +++ b/lib/core.ps1 @@ -63,6 +63,18 @@ function Optimize-SecurityProtocol { } } +function Get-Encoding($wc) { + if ($null -ne $wc.ResponseHeaders -and $wc.ResponseHeaders['Content-Type'] -match 'charset=([^;]*)') { + return [System.Text.Encoding]::GetEncoding($Matches[1]) + } else { + return [System.Text.Encoding]::GetEncoding('utf-8') + } +} + +function Get-UserAgent() { + return "Scoop/1.0 (+http://scoop.sh/) PowerShell/$($PSVersionTable.PSVersion.Major).$($PSVersionTable.PSVersion.Minor) (Windows NT $([System.Environment]::OSVersion.Version.Major).$([System.Environment]::OSVersion.Version.Minor); $(if(${env:ProgramFiles(Arm)}){'ARM64; '}elseif($env:PROCESSOR_ARCHITECTURE -eq 'AMD64'){'Win64; x64; '})$(if($env:PROCESSOR_ARCHITEW6432 -in 'AMD64','ARM64'){'WOW64; '})$PSEdition)" +} + function Show-DeprecatedWarning { <# .SYNOPSIS @@ -216,6 +228,35 @@ function Complete-ConfigChange { } } +function setup_proxy() { + # note: '@' and ':' in password must be escaped, e.g. 'p@ssword' -> p\@ssword' + $proxy = get_config PROXY + if(!$proxy) { + return + } + try { + $credentials, $address = $proxy -split '(?/dev/null"', - 'if %errorlevel% equ 0 (', - " bash `"`$(wslpath -u '$resolved_path')`" $quoted_arg %*", - ') else (', - " set args=$quoted_arg %*", - ' setlocal enabledelayedexpansion', - ' if not "!args!"=="" set args=!args:"=""!', - " bash -c `"`$(cygpath -u '$resolved_path') !args!`"", + "@bash `"`$(wslpath -u '$resolved_path')`" $arg %* 2>nul", + '@if %errorlevel% neq 0 (', + " @bash `"`$(cygpath -u '$resolved_path')`" $arg %* 2>nul", ')' ) -join "`r`n" | Out-UTF8File "$shim.cmd" @@ -1224,6 +1282,112 @@ function substitute($entity, [Hashtable] $params, [Bool]$regexEscape = $false) { return $newentity } +function format_hash([String] $hash) { + $hash = $hash.toLower() + switch ($hash.Length) + { + 32 { $hash = "md5:$hash" } # md5 + 40 { $hash = "sha1:$hash" } # sha1 + 64 { $hash = $hash } # sha256 + 128 { $hash = "sha512:$hash" } # sha512 + default { $hash = $null } + } + return $hash +} + +function format_hash_aria2([String] $hash) { + $hash = $hash -split ':' | Select-Object -Last 1 + switch ($hash.Length) + { + 32 { $hash = "md5=$hash" } # md5 + 40 { $hash = "sha-1=$hash" } # sha1 + 64 { $hash = "sha-256=$hash" } # sha256 + 128 { $hash = "sha-512=$hash" } # sha512 + default { $hash = $null } + } + return $hash +} + +function get_hash([String] $multihash) { + $type, $hash = $multihash -split ':' + if(!$hash) { + # no type specified, assume sha256 + $type, $hash = 'sha256', $multihash + } + + if(@('md5','sha1','sha256', 'sha512') -notcontains $type) { + return $null, "Hash type '$type' isn't supported." + } + + return $type, $hash.ToLower() +} + +function Get-GitHubToken { + return $env:SCOOP_GH_TOKEN, (get_config GH_TOKEN) | Where-Object -Property Length -Value 0 -GT | Select-Object -First 1 +} + +function handle_special_urls($url) +{ + # FossHub.com + if ($url -match "^(?:.*fosshub.com\/)(?.*)(?:\/|\?dwl=)(?.*)$") { + $Body = @{ + projectUri = $Matches.name; + fileName = $Matches.filename; + source = 'CF'; + isLatestVersion = $true + } + if ((Invoke-RestMethod -Uri $url) -match '"p":"(?[a-f0-9]{24}).*?"r":"(?[a-f0-9]{24})') { + $Body.Add("projectId", $Matches.pid) + $Body.Add("releaseId", $Matches.rid) + } + $url = Invoke-RestMethod -Method Post -Uri "https://api.fosshub.com/download/" -ContentType "application/json" -Body (ConvertTo-Json $Body -Compress) + if ($null -eq $url.error) { + $url = $url.data.url + } + } + + # Sourceforge.net + if ($url -match "(?:downloads\.)?sourceforge.net\/projects?\/(?[^\/]+)\/(?:files\/)?(?.*?)(?:$|\/download|\?)") { + # Reshapes the URL to avoid redirections + $url = "https://downloads.sourceforge.net/project/$($matches['project'])/$($matches['file'])" + } + + # Github.com + if ($url -match 'github.com/(?[^/]+)/(?[^/]+)/releases/download/(?[^/]+)/(?[^/#]+)(?.*)' -and ($token = Get-GitHubToken)) { + $headers = @{ "Authorization" = "token $token" } + $privateUrl = "https://api.github.com/repos/$($Matches.owner)/$($Matches.repo)" + $assetUrl = "https://api.github.com/repos/$($Matches.owner)/$($Matches.repo)/releases/tags/$($Matches.tag)" + + if ((Invoke-RestMethod -Uri $privateUrl -Headers $headers).Private) { + $url = ((Invoke-RestMethod -Uri $assetUrl -Headers $headers).Assets | Where-Object -Property Name -EQ -Value $Matches.file).Url, $Matches.filename -join '' + } + } + + return $url +} + +function get_magic_bytes($file) { + if(!(Test-Path $file)) { + return '' + } + + if((Get-Command Get-Content).parameters.ContainsKey('AsByteStream')) { + # PowerShell Core (6.0+) '-Encoding byte' is replaced by '-AsByteStream' + return Get-Content $file -AsByteStream -TotalCount 8 + } + else { + return Get-Content $file -Encoding byte -TotalCount 8 + } +} + +function get_magic_bytes_pretty($file, $glue = ' ') { + if(!(Test-Path $file)) { + return '' + } + + return (get_magic_bytes $file | ForEach-Object { $_.ToString('x2') }) -join $glue +} + function Out-UTF8File { param( [Parameter(Mandatory = $True, Position = 0)] @@ -1309,3 +1473,6 @@ $scoopPathEnvVar = switch (get_config USE_ISOLATED_PATH) { # OS information $WindowsBuild = [System.Environment]::OSVersion.Version.Build + +# Setup proxy globally +setup_proxy diff --git a/lib/decompress.ps1 b/lib/decompress.ps1 index 3f174bf0c3..9c3f5c6813 100644 --- a/lib/decompress.ps1 +++ b/lib/decompress.ps1 @@ -24,7 +24,7 @@ function Invoke-Extraction { $extractFn = $null switch -regex ($Name[$i]) { '\.zip$' { - if ((Test-HelperInstalled -Helper 7zip) -or ((get_config USE_EXTERNAL_7ZIP) -and (Test-CommandAvailable 7z))) { + if ((Test-HelperInstalled -Helper 7zip) -or ((get_config 7ZIPEXTRACT_USE_EXTERNAL) -and (Test-CommandAvailable 7z))) { $extractFn = 'Expand-7zipArchive' } else { $extractFn = 'Expand-ZipArchive' @@ -123,18 +123,15 @@ function Expand-7zipArchive { } if (!$IsTar -and $ExtractDir) { movedir "$DestinationPath\$ExtractDir" $DestinationPath | Out-Null - # Remove temporary directory if it is empty - $ExtractDirTopPath = [string] "$DestinationPath\$($ExtractDir -replace '[\\/].*')" - if ((Get-ChildItem -Path $ExtractDirTopPath -Force -ErrorAction Ignore).Count -eq 0) { - Remove-Item -Path $ExtractDirTopPath -Recurse -Force -ErrorAction Ignore - } + # Remove temporary directory + Remove-Item "$DestinationPath\$($ExtractDir -replace '[\\/].*')" -Recurse -Force -ErrorAction Ignore } if (Test-Path $LogPath) { Remove-Item $LogPath -Force } if ($Removal) { if (($Path -replace '.*\.([^\.]*)$', '$1') -eq '001') { - # Remove splitted 7-zip archive parts + # Remove splited 7-zip archive parts Get-ChildItem "$($Path -replace '\.[^\.]*$', '').???" | Remove-Item -Force } elseif (($Path -replace '.*\.part(\d+)\.rar$', '$1')[-1] -eq '1') { # Remove splitted RAR archive parts diff --git a/lib/download.ps1 b/lib/download.ps1 deleted file mode 100644 index 70641cca7f..0000000000 --- a/lib/download.ps1 +++ /dev/null @@ -1,769 +0,0 @@ -# Description: Functions for downloading files - -## Meta downloader - -function Invoke-ScoopDownload ($app, $version, $manifest, $bucket, $architecture, $dir, $use_cache = $true, $check_hash = $true) { - # we only want to show this warning once - if (!$use_cache) { warn 'Cache is being ignored.' } - - # can be multiple urls: if there are, then installer should go first to make 'installer.args' section work - $urls = @(script:url $manifest $architecture) - - # can be multiple cookies: they will be used for all HTTP requests. - $cookies = $manifest.cookie - - # download first - if (Test-Aria2Enabled) { - Invoke-CachedAria2Download $app $version $manifest $architecture $dir $cookies $use_cache $check_hash - } else { - foreach ($url in $urls) { - $fname = url_filename $url - - try { - Invoke-CachedDownload $app $version $url "$dir\$fname" $cookies $use_cache - } catch { - Write-Host -ForegroundColor DarkRed $_ - abort "URL $url is not valid" - } - - if ($check_hash) { - $manifest_hash = hash_for_url $manifest $url $architecture - $ok, $err = check_hash "$dir\$fname" $manifest_hash $(show_app $app $bucket) - if (!$ok) { - error $err - $cached = cache_path $app $version $url - if (Test-Path $cached) { - # rm cached file - Remove-Item -Force $cached - } - if ($url.Contains('sourceforge.net')) { - Write-Host -ForegroundColor Yellow 'SourceForge.net is known for causing hash validation fails. Please try again before opening a ticket.' - } - abort $(new_issue_msg $app $bucket 'hash check failed') - } - } - } - } - - return $urls.ForEach({ url_filename $_ }) -} - -## [System.Net] downloader - -function Invoke-CachedDownload ($app, $version, $url, $to, $cookies = $null, $use_cache = $true) { - $cached = cache_path $app $version $url - - if (!(Test-Path $cached) -or !$use_cache) { - ensure $cachedir | Out-Null - Start-Download $url "$cached.download" $cookies - Move-Item "$cached.download" $cached -Force - } else { Write-Host "Loading $(url_remote_filename $url) from cache" } - - if (!($null -eq $to)) { - if ($use_cache) { - Copy-Item $cached $to - } else { - Move-Item $cached $to -Force - } - } -} - -function Start-Download ($url, $to, $cookies) { - $progress = [console]::isoutputredirected -eq $false -and - $host.name -ne 'Windows PowerShell ISE Host' - - try { - $url = handle_special_urls $url - Invoke-Download $url $to $cookies $progress - } catch { - $e = $_.exception - if ($e.Response.StatusCode -eq 'Unauthorized') { - warn 'Token might be misconfigured.' - } - if ($e.innerexception) { $e = $e.innerexception } - throw $e - } -} - -function Invoke-Download ($url, $to, $cookies, $progress) { - # download with filesize and progress indicator - $reqUrl = ($url -split '#')[0] - $wreq = [Net.WebRequest]::Create($reqUrl) - if ($wreq -is [Net.HttpWebRequest]) { - $wreq.UserAgent = Get-UserAgent - if (-not ($url -match 'sourceforge\.net' -or $url -match 'portableapps\.com')) { - $wreq.Referer = strip_filename $url - } - if ($url -match 'api\.github\.com/repos') { - $wreq.Accept = 'application/octet-stream' - $wreq.Headers['Authorization'] = "Bearer $(Get-GitHubToken)" - $wreq.Headers['X-GitHub-Api-Version'] = '2022-11-28' - } - if ($cookies) { - $wreq.Headers.Add('Cookie', (cookie_header $cookies)) - } - - get_config PRIVATE_HOSTS | Where-Object { $_ -ne $null -and $url -match $_.match } | ForEach-Object { - (ConvertFrom-StringData -StringData $_.Headers).GetEnumerator() | ForEach-Object { - $wreq.Headers[$_.Key] = $_.Value - } - } - } - - try { - $wres = $wreq.GetResponse() - } catch [System.Net.WebException] { - $exc = $_.Exception - $handledCodes = @( - [System.Net.HttpStatusCode]::MovedPermanently, # HTTP 301 - [System.Net.HttpStatusCode]::Found, # HTTP 302 - [System.Net.HttpStatusCode]::SeeOther, # HTTP 303 - [System.Net.HttpStatusCode]::TemporaryRedirect # HTTP 307 - ) - - # Only handle redirection codes - $redirectRes = $exc.Response - if ($handledCodes -notcontains $redirectRes.StatusCode) { - throw $exc - } - - # Get the new location of the file - if ((-not $redirectRes.Headers) -or ($redirectRes.Headers -notcontains 'Location')) { - throw $exc - } - - $newUrl = $redirectRes.Headers['Location'] - info "Following redirect to $newUrl..." - - # Handle manual file rename - if ($url -like '*#/*') { - $null, $postfix = $url -split '#/' - $newUrl = "$newUrl`#/$postfix" - } - - Invoke-Download $newUrl $to $cookies $progress - return - } - - $total = $wres.ContentLength - if ($total -eq -1 -and $wreq -is [net.ftpwebrequest]) { - $total = ftp_file_size($url) - } - - if ($progress -and ($total -gt 0)) { - [console]::CursorVisible = $false - function Trace-DownloadProgress ($read) { - Write-DownloadProgress $read $total $url - } - } else { - Write-Host "Downloading $url ($(filesize $total))..." - function Trace-DownloadProgress { - #no op - } - } - - try { - $s = $wres.getresponsestream() - $fs = [io.file]::openwrite($to) - $buffer = New-Object byte[] 2048 - $totalRead = 0 - $sw = [diagnostics.stopwatch]::StartNew() - - Trace-DownloadProgress $totalRead - while (($read = $s.read($buffer, 0, $buffer.length)) -gt 0) { - $fs.write($buffer, 0, $read) - $totalRead += $read - if ($sw.elapsedmilliseconds -gt 100) { - $sw.restart() - Trace-DownloadProgress $totalRead - } - } - $sw.stop() - Trace-DownloadProgress $totalRead - } finally { - if ($progress) { - [console]::CursorVisible = $true - Write-Host - } - if ($fs) { - $fs.close() - } - if ($s) { - $s.close() - } - $wres.close() - } -} - -function Format-DownloadProgress ($url, $read, $total, $console) { - $filename = url_remote_filename $url - - # calculate current percentage done - $p = [math]::Round($read / $total * 100, 0) - - # pre-generate LHS and RHS of progress string - # so we know how much space we have - $left = "$filename ($(filesize $total))" - $right = [string]::Format('{0,3}%', $p) - - # calculate remaining width for progress bar - $midwidth = $console.BufferSize.Width - ($left.Length + $right.Length + 8) - - # calculate how many characters are completed - $completed = [math]::Abs([math]::Round(($p / 100) * $midwidth, 0) - 1) - - # generate dashes to symbolise completed - if ($completed -gt 1) { - $dashes = [string]::Join('', ((1..$completed) | ForEach-Object { '=' })) - } - - # this is why we calculate $completed - 1 above - $dashes += switch ($p) { - 100 { '=' } - default { '>' } - } - - # the remaining characters are filled with spaces - $spaces = switch ($dashes.Length) { - $midwidth { [string]::Empty } - default { - [string]::Join('', ((1..($midwidth - $dashes.Length)) | ForEach-Object { ' ' })) - } - } - - "$left [$dashes$spaces] $right" -} - -function Write-DownloadProgress ($read, $total, $url) { - $console = $Host.UI.RawUI - $left = $console.CursorPosition.X - $top = $console.CursorPosition.Y - $width = $console.BufferSize.Width - - if ($read -eq 0) { - $maxOutputLength = $(Format-DownloadProgress $url 100 $total $console).Length - if (($left + $maxOutputLength) -gt $width) { - # not enough room to print progress on this line - # print on new line - Write-Host - $left = 0 - $top = $top + 1 - if ($top -gt $console.CursorPosition.Y) { $top = $console.CursorPosition.Y } - } - } - - Write-Host $(Format-DownloadProgress $url $read $total $console) -NoNewline - [console]::SetCursorPosition($left, $top) -} - -## Aria2 downloader - -function Test-Aria2Enabled { - return (Test-HelperInstalled -Helper Aria2) -and (get_config 'aria2-enabled' $true) -} - -function aria_exit_code($exitcode) { - $codes = @{ - 0 = 'All downloads were successful' - 1 = 'An unknown error occurred' - 2 = 'Timeout' - 3 = 'Resource was not found' - 4 = 'Aria2 saw the specified number of "resource not found" error. See --max-file-not-found option' - 5 = 'Download aborted because download speed was too slow. See --lowest-speed-limit option' - 6 = 'Network problem occurred.' - 7 = 'There were unfinished downloads. This error is only reported if all finished downloads were successful and there were unfinished downloads in a queue when aria2 exited by pressing Ctrl-C by an user or sending TERM or INT signal' - 8 = 'Remote server did not support resume when resume was required to complete download' - 9 = 'There was not enough disk space available' - 10 = 'Piece length was different from one in .aria2 control file. See --allow-piece-length-change option' - 11 = 'Aria2 was downloading same file at that moment' - 12 = 'Aria2 was downloading same info hash torrent at that moment' - 13 = 'File already existed. See --allow-overwrite option' - 14 = 'Renaming file failed. See --auto-file-renaming option' - 15 = 'Aria2 could not open existing file' - 16 = 'Aria2 could not create new file or truncate existing file' - 17 = 'File I/O error occurred' - 18 = 'Aria2 could not create directory' - 19 = 'Name resolution failed' - 20 = 'Aria2 could not parse Metalink document' - 21 = 'FTP command failed' - 22 = 'HTTP response header was bad or unexpected' - 23 = 'Too many redirects occurred' - 24 = 'HTTP authorization failed' - 25 = 'Aria2 could not parse bencoded file (usually ".torrent" file)' - 26 = '".torrent" file was corrupted or missing information that aria2 needed' - 27 = 'Magnet URI was bad' - 28 = 'Bad/unrecognized option was given or unexpected option argument was given' - 29 = 'The remote server was unable to handle the request due to a temporary overloading or maintenance' - 30 = 'Aria2 could not parse JSON-RPC request' - 31 = 'Reserved. Not used' - 32 = 'Checksum validation failed' - } - if ($null -eq $codes[$exitcode]) { - return 'An unknown error occurred' - } - return $codes[$exitcode] -} - -function get_filename_from_metalink($file) { - $bytes = get_magic_bytes_pretty $file '' - # check if file starts with ' p\@ssword' - $proxy = get_config PROXY - if (!$proxy) { - return - } - try { - $credentials, $address = $proxy -split '(?'." - } - $ret -} - -### URL handling - -function handle_special_urls($url) { - # FossHub.com - if ($url -match '^(?:.*fosshub.com\/)(?.*)(?:\/|\?dwl=)(?.*)$') { - $Body = @{ - projectUri = $Matches.name - fileName = $Matches.filename - source = 'CF' - isLatestVersion = $true - } - if ((Invoke-RestMethod -Uri $url) -match '"p":"(?[a-f0-9]{24}).*?"r":"(?[a-f0-9]{24})') { - $Body.Add('projectId', $Matches.pid) - $Body.Add('releaseId', $Matches.rid) - } - $url = Invoke-RestMethod -Method Post -Uri 'https://api.fosshub.com/download/' -ContentType 'application/json' -Body (ConvertTo-Json $Body -Compress) - if ($null -eq $url.error) { - $url = $url.data.url - } - } - - # Sourceforge.net - if ($url -match '(?:downloads\.)?sourceforge.net\/projects?\/(?[^\/]+)\/(?:files\/)?(?.*?)(?:$|\/download|\?)') { - # Reshapes the URL to avoid redirections - $url = "https://downloads.sourceforge.net/project/$($matches['project'])/$($matches['file'])" - } - - # Github.com - if ($url -match 'github.com/(?[^/]+)/(?[^/]+)/releases/download/(?[^/]+)/(?[^/#]+)(?.*)' -and ($token = Get-GitHubToken)) { - $headers = @{ 'Authorization' = "token $token" } - $privateUrl = "https://api.github.com/repos/$($Matches.owner)/$($Matches.repo)" - $assetUrl = "https://api.github.com/repos/$($Matches.owner)/$($Matches.repo)/releases/tags/$($Matches.tag)" - - if ((Invoke-RestMethod -Uri $privateUrl -Headers $headers).Private) { - $url = ((Invoke-RestMethod -Uri $assetUrl -Headers $headers).Assets | Where-Object -Property Name -EQ -Value $Matches.file).Url, $Matches.filename -join '' - } - } - - return $url -} - -### Remote file information - -function download_json($url) { - $githubtoken = Get-GitHubToken - $authheader = @{} - if ($githubtoken) { - $authheader = @{'Authorization' = "token $githubtoken" } - } - $ProgressPreference = 'SilentlyContinue' - $result = Invoke-WebRequest $url -UseBasicParsing -Headers $authheader | Select-Object -ExpandProperty content | ConvertFrom-Json - $ProgressPreference = 'Continue' - $result -} - -function get_magic_bytes($file) { - if (!(Test-Path $file)) { - return '' - } - - if ((Get-Command Get-Content).parameters.ContainsKey('AsByteStream')) { - # PowerShell Core (6.0+) '-Encoding byte' is replaced by '-AsByteStream' - return Get-Content $file -AsByteStream -TotalCount 8 - } else { - return Get-Content $file -Encoding byte -TotalCount 8 - } -} - -function get_magic_bytes_pretty($file, $glue = ' ') { - if (!(Test-Path $file)) { - return '' - } - - return (get_magic_bytes $file | ForEach-Object { $_.ToString('x2') }) -join $glue -} - -Function Get-RemoteFileSize ($Uri) { - $response = Invoke-WebRequest -Uri $Uri -Method HEAD -UseBasicParsing - if (!$response.Headers.StatusCode) { - $response.Headers.'Content-Length' | ForEach-Object { [int]$_ } - } -} - -function ftp_file_size($url) { - $request = [net.ftpwebrequest]::create($url) - $request.method = [net.webrequestmethods+ftp]::getfilesize - $request.getresponse().contentlength -} - -function url_filename($url) { - (Split-Path $url -Leaf).split('?') | Select-Object -First 1 -} - -function url_remote_filename($url) { - # Unlike url_filename which can be tricked by appending a - # URL fragment (e.g. #/dl.7z, useful for coercing a local filename), - # this function extracts the original filename from the URL. - $uri = (New-Object URI $url) - $basename = Split-Path $uri.PathAndQuery -Leaf - If ($basename -match '.*[?=]+([\w._-]+)') { - $basename = $matches[1] - } - If (($basename -notlike '*.*') -or ($basename -match '^[v.\d]+$')) { - $basename = Split-Path $uri.AbsolutePath -Leaf - } - If (($basename -notlike '*.*') -and ($uri.Fragment -ne '')) { - $basename = $uri.Fragment.Trim('/', '#') - } - return $basename -} - -### Hash-related functions - -function hash_for_url($manifest, $url, $arch) { - $hashes = @(hash $manifest $arch) | Where-Object { $_ -ne $null } - - if ($hashes.length -eq 0) { return $null } - - $urls = @(script:url $manifest $arch) - - $index = [array]::IndexOf($urls, $url) - if ($index -eq -1) { abort "Couldn't find hash in manifest for '$url'." } - - @($hashes)[$index] -} - -function check_hash($file, $hash, $app_name) { - # returns (ok, err) - if (!$hash) { - warn "Warning: No hash in manifest. SHA256 for '$(fname $file)' is:`n $((Get-FileHash -Path $file -Algorithm SHA256).Hash.ToLower())" - return $true, $null - } - - Write-Host 'Checking hash of ' -NoNewline - Write-Host $(url_remote_filename $url) -ForegroundColor Cyan -NoNewline - Write-Host ' ... ' -NoNewline - $algorithm, $expected = get_hash $hash - if ($null -eq $algorithm) { - return $false, "Hash type '$algorithm' isn't supported." - } - - $actual = (Get-FileHash -Path $file -Algorithm $algorithm).Hash.ToLower() - $expected = $expected.ToLower() - - if ($actual -ne $expected) { - $msg = "Hash check failed!`n" - $msg += "App: $app_name`n" - $msg += "URL: $url`n" - if (Test-Path $file) { - $msg += "First bytes: $((get_magic_bytes_pretty $file ' ').ToUpper())`n" - } - if ($expected -or $actual) { - $msg += "Expected: $expected`n" - $msg += "Actual: $actual" - } - return $false, $msg - } - Write-Host 'ok.' -f Green - return $true, $null -} - -function get_hash([String] $multihash) { - $type, $hash = $multihash -split ':' - if (!$hash) { - # no type specified, assume sha256 - $type, $hash = 'sha256', $multihash - } - - if (@('md5', 'sha1', 'sha256', 'sha512') -notcontains $type) { - return $null, "Hash type '$type' isn't supported." - } - - return $type, $hash.ToLower() -} - -# Setup proxy globally -setup_proxy diff --git a/lib/install.ps1 b/lib/install.ps1 index 56cfdac924..54b0922e26 100644 --- a/lib/install.ps1 +++ b/lib/install.ps1 @@ -81,10 +81,576 @@ function install_app($app, $architecture, $global, $suggested, $use_cache = $tru show_notes $manifest $dir $original_dir $persist_dir } +function Invoke-CachedDownload ($app, $version, $url, $to, $cookies = $null, $use_cache = $true) { + $cached = cache_path $app $version $url + + if (!(Test-Path $cached) -or !$use_cache) { + ensure $cachedir | Out-Null + Start-Download $url "$cached.download" $cookies + Move-Item "$cached.download" $cached -Force + } else { Write-Host "Loading $(url_remote_filename $url) from cache" } + + if (!($null -eq $to)) { + if ($use_cache) { + Copy-Item $cached $to + } else { + Move-Item $cached $to -Force + } + } +} + +function Start-Download ($url, $to, $cookies) { + $progress = [console]::isoutputredirected -eq $false -and + $host.name -ne 'Windows PowerShell ISE Host' + + try { + $url = handle_special_urls $url + Invoke-Download $url $to $cookies $progress + } catch { + $e = $_.exception + if ($e.Response.StatusCode -eq 'Unauthorized') { + warn 'Token might be misconfigured.' + } + if ($e.innerexception) { $e = $e.innerexception } + throw $e + } +} + +function aria_exit_code($exitcode) { + $codes = @{ + 0 = 'All downloads were successful' + 1 = 'An unknown error occurred' + 2 = 'Timeout' + 3 = 'Resource was not found' + 4 = 'Aria2 saw the specified number of "resource not found" error. See --max-file-not-found option' + 5 = 'Download aborted because download speed was too slow. See --lowest-speed-limit option' + 6 = 'Network problem occurred.' + 7 = 'There were unfinished downloads. This error is only reported if all finished downloads were successful and there were unfinished downloads in a queue when aria2 exited by pressing Ctrl-C by an user or sending TERM or INT signal' + 8 = 'Remote server did not support resume when resume was required to complete download' + 9 = 'There was not enough disk space available' + 10 = 'Piece length was different from one in .aria2 control file. See --allow-piece-length-change option' + 11 = 'Aria2 was downloading same file at that moment' + 12 = 'Aria2 was downloading same info hash torrent at that moment' + 13 = 'File already existed. See --allow-overwrite option' + 14 = 'Renaming file failed. See --auto-file-renaming option' + 15 = 'Aria2 could not open existing file' + 16 = 'Aria2 could not create new file or truncate existing file' + 17 = 'File I/O error occurred' + 18 = 'Aria2 could not create directory' + 19 = 'Name resolution failed' + 20 = 'Aria2 could not parse Metalink document' + 21 = 'FTP command failed' + 22 = 'HTTP response header was bad or unexpected' + 23 = 'Too many redirects occurred' + 24 = 'HTTP authorization failed' + 25 = 'Aria2 could not parse bencoded file (usually ".torrent" file)' + 26 = '".torrent" file was corrupted or missing information that aria2 needed' + 27 = 'Magnet URI was bad' + 28 = 'Bad/unrecognized option was given or unexpected option argument was given' + 29 = 'The remote server was unable to handle the request due to a temporary overloading or maintenance' + 30 = 'Aria2 could not parse JSON-RPC request' + 31 = 'Reserved. Not used' + 32 = 'Checksum validation failed' + } + if ($null -eq $codes[$exitcode]) { + return 'An unknown error occurred' + } + return $codes[$exitcode] +} + +function get_filename_from_metalink($file) { + $bytes = get_magic_bytes_pretty $file '' + # check if file starts with '' } + } + + # the remaining characters are filled with spaces + $spaces = switch ($dashes.Length) { + $midwidth { [string]::Empty } + default { + [string]::Join('', ((1..($midwidth - $dashes.Length)) | ForEach-Object { ' ' })) + } + } + + "$left [$dashes$spaces] $right" +} + +function Write-DownloadProgress ($read, $total, $url) { + $console = $host.UI.RawUI + $left = $console.CursorPosition.X + $top = $console.CursorPosition.Y + $width = $console.BufferSize.Width + + if ($read -eq 0) { + $maxOutputLength = $(Format-DownloadProgress $url 100 $total $console).length + if (($left + $maxOutputLength) -gt $width) { + # not enough room to print progress on this line + # print on new line + Write-Host + $left = 0 + $top = $top + 1 + if ($top -gt $console.CursorPosition.Y) { $top = $console.CursorPosition.Y } + } + } + + Write-Host $(Format-DownloadProgress $url $read $total $console) -NoNewline + [console]::SetCursorPosition($left, $top) +} + +function Invoke-ScoopDownload ($app, $version, $manifest, $bucket, $architecture, $dir, $use_cache = $true, $check_hash = $true) { + # we only want to show this warning once + if (!$use_cache) { warn 'Cache is being ignored.' } + + # can be multiple urls: if there are, then installer should go first to make 'installer.args' section work + $urls = @(script:url $manifest $architecture) + + # can be multiple cookies: they will be used for all HTTP requests. + $cookies = $manifest.cookie + + # download first + if (Test-Aria2Enabled) { + Invoke-CachedAria2Download $app $version $manifest $architecture $dir $cookies $use_cache $check_hash + } else { + foreach ($url in $urls) { + $fname = url_filename $url + + try { + Invoke-CachedDownload $app $version $url "$dir\$fname" $cookies $use_cache + } catch { + Write-Host -f darkred $_ + abort "URL $url is not valid" + } + + if ($check_hash) { + $manifest_hash = hash_for_url $manifest $url $architecture + $ok, $err = check_hash "$dir\$fname" $manifest_hash $(show_app $app $bucket) + if (!$ok) { + error $err + $cached = cache_path $app $version $url + if (Test-Path $cached) { + # rm cached file + Remove-Item -Force $cached + } + if ($url.Contains('sourceforge.net')) { + Write-Host -f yellow 'SourceForge.net is known for causing hash validation fails. Please try again before opening a ticket.' + } + abort $(new_issue_msg $app $bucket 'hash check failed') + } + } + } + } + + return $urls.ForEach({ url_filename $_ }) +} + +function cookie_header($cookies) { + if (!$cookies) { return } + + $vals = $cookies.psobject.properties | ForEach-Object { + "$($_.name)=$($_.value)" + } + + [string]::join(';', $vals) +} + function is_in_dir($dir, $check) { $check -match "^$([regex]::Escape("$dir"))([/\\]|$)" } +function ftp_file_size($url) { + $request = [net.ftpwebrequest]::create($url) + $request.method = [net.webrequestmethods+ftp]::getfilesize + $request.getresponse().contentlength +} + +# hashes +function hash_for_url($manifest, $url, $arch) { + $hashes = @(hash $manifest $arch) | Where-Object { $_ -ne $null } + + if ($hashes.length -eq 0) { return $null } + + $urls = @(script:url $manifest $arch) + + $index = [array]::indexof($urls, $url) + if ($index -eq -1) { abort "Couldn't find hash in manifest for '$url'." } + + @($hashes)[$index] +} + +# returns (ok, err) +function check_hash($file, $hash, $app_name) { + if (!$hash) { + warn "Warning: No hash in manifest. SHA256 for '$(fname $file)' is:`n $((Get-FileHash -Path $file -Algorithm SHA256).Hash.ToLower())" + return $true, $null + } + + Write-Host 'Checking hash of ' -NoNewline + Write-Host $(url_remote_filename $url) -f Cyan -NoNewline + Write-Host ' ... ' -NoNewline + $algorithm, $expected = get_hash $hash + if ($null -eq $algorithm) { + return $false, "Hash type '$algorithm' isn't supported." + } + + $actual = (Get-FileHash -Path $file -Algorithm $algorithm).Hash.ToLower() + $expected = $expected.ToLower() + + if ($actual -ne $expected) { + $msg = "Hash check failed!`n" + $msg += "App: $app_name`n" + $msg += "URL: $url`n" + if (Test-Path $file) { + $msg += "First bytes: $((get_magic_bytes_pretty $file ' ').ToUpper())`n" + } + if ($expected -or $actual) { + $msg += "Expected: $expected`n" + $msg += "Actual: $actual" + } + return $false, $msg + } + Write-Host 'ok.' -f Green + return $true, $null +} + function Invoke-Installer { [CmdletBinding()] param ( diff --git a/lib/manifest.ps1 b/lib/manifest.ps1 index 29c898b6a7..9ca618158b 100644 --- a/lib/manifest.ps1 +++ b/lib/manifest.ps1 @@ -40,74 +40,30 @@ function Get-Manifest($app) { $app = appname_from_url $url $manifest = url_manifest $url } else { - # Check if the manifest is already installed - if (installed $app) { - $global = installed $app $true - $ver = Select-CurrentVersion -AppName $app -Global:$global - if (!$ver) { - $app, $bucket, $ver = parse_app $app - $ver = Select-CurrentVersion -AppName $app -Global:$global - } - $install_info_path = "$(versiondir $app $ver $global)\install.json" - if (Test-Path $install_info_path) { - $install_info = parse_json $install_info_path - $bucket = $install_info.bucket - if (!$bucket) { - $url = $install_info.url - if ($url -match '^(ht|f)tps?://|\\\\') { - $manifest = url_manifest $url - } - if (!$manifest) { - if (Test-Path $url) { - $manifest = parse_json $url - } else { - # Fallback to installed manifest - $manifest = installed_manifest $app $ver $global - } - } - } else { - $manifest = manifest $app $bucket - if (!$manifest) { - $deprecated_dir = (Find-BucketDirectory -Name $bucket -Root) + '\deprecated' - $manifest = parse_json (Get-ChildItem $deprecated_dir -Filter "$(sanitary_path $app).json" -Recurse).FullName - } - } - } + $app, $bucket, $version = parse_app $app + if ($bucket) { + $manifest = manifest $app $bucket } else { - $app, $bucket, $version = parse_app $app - if ($bucket) { - $manifest = manifest $app $bucket - } else { - $matched_buckets = @() - foreach ($tekcub in Get-LocalBucket) { - $current_manifest = manifest $app $tekcub - if (!$manifest -and $current_manifest) { - $manifest = $current_manifest - $bucket = $tekcub - } - if ($current_manifest) { - $matched_buckets += $tekcub - } + foreach ($tekcub in Get-LocalBucket) { + $manifest = manifest $app $tekcub + if ($manifest) { + $bucket = $tekcub + break } } - if (!$manifest) { - # couldn't find app in buckets: check if it's a local path - if (Test-Path $app) { - $url = Convert-Path $app - $app = appname_from_url $url - $manifest = parse_json $url - } else { - if (($app -match '\\/') -or $app.EndsWith('.json')) { $url = $app } - $app = appname_from_url $app - } + } + if (!$manifest) { + # couldn't find app in buckets: check if it's a local path + if (Test-Path $app) { + $url = Convert-Path $app + $app = appname_from_url $url + $manifest = url_manifest $url + } else { + if (($app -match '\\/') -or $app.EndsWith('.json')) { $url = $app } + $app = appname_from_url $app } } } - - if ($matched_buckets.Length -gt 1) { - warn "Multiple buckets contain manifest '$app', the current selection is '$bucket/$app'." - } - return $app, $manifest, $bucket, $url } diff --git a/libexec/scoop-bucket.ps1 b/libexec/scoop-bucket.ps1 index d3b7728559..ceb28865a5 100644 --- a/libexec/scoop-bucket.ps1 +++ b/libexec/scoop-bucket.ps1 @@ -19,10 +19,6 @@ # scoop bucket known param($cmd, $name, $repo) -if (get_config NO_JUNCTION) { - . "$PSScriptRoot\..\lib\versions.ps1" -} - if (get_config USE_SQLITE_CACHE) { . "$PSScriptRoot\..\lib\manifest.ps1" . "$PSScriptRoot\..\lib\database.ps1" diff --git a/libexec/scoop-cat.ps1 b/libexec/scoop-cat.ps1 index fc85fced92..5cf363162d 100644 --- a/libexec/scoop-cat.ps1 +++ b/libexec/scoop-cat.ps1 @@ -7,9 +7,7 @@ param($app) . "$PSScriptRoot\..\lib\json.ps1" # 'ConvertToPrettyJson' -. "$PSScriptRoot\..\lib\versions.ps1" # 'Select-CurrentVersion' . "$PSScriptRoot\..\lib\manifest.ps1" # 'Get-Manifest' -. "$PSScriptRoot\..\lib\download.ps1" # 'Get-UserAgent' if (!$app) { error ' missing'; my_usage; exit 1 } diff --git a/libexec/scoop-depends.ps1 b/libexec/scoop-depends.ps1 index 25ed614ca3..414d1b7113 100644 --- a/libexec/scoop-depends.ps1 +++ b/libexec/scoop-depends.ps1 @@ -3,9 +3,7 @@ . "$PSScriptRoot\..\lib\getopt.ps1" . "$PSScriptRoot\..\lib\depends.ps1" # 'Get-Dependency' -. "$PSScriptRoot\..\lib\versions.ps1" # 'Select-CurrentVersion' . "$PSScriptRoot\..\lib\manifest.ps1" # 'Get-Manifest' (indirectly) -. "$PSScriptRoot\..\lib\download.ps1" # 'Get-UserAgent' $opt, $apps, $err = getopt $args 'a:' 'arch=' $app = $apps[0] @@ -22,14 +20,7 @@ try { $deps = @() Get-Dependency $app $architecture | ForEach-Object { $dep = [ordered]@{} - - $app, $null, $bucket, $url = Get-Manifest $_ - if (!$url) { - $bucket, $app = $_ -split '/' - } - $dep.Source = if ($url) { $url } else { $bucket } - $dep.Name = $app - + $dep.Source, $dep.Name = $_ -split '/' $deps += [PSCustomObject]$dep } $deps diff --git a/libexec/scoop-download.ps1 b/libexec/scoop-download.ps1 index 996cb4e8e6..1901527b09 100644 --- a/libexec/scoop-download.ps1 +++ b/libexec/scoop-download.ps1 @@ -22,9 +22,8 @@ . "$PSScriptRoot\..\lib\getopt.ps1" . "$PSScriptRoot\..\lib\json.ps1" # 'autoupdate.ps1' (indirectly) . "$PSScriptRoot\..\lib\autoupdate.ps1" # 'generate_user_manifest' (indirectly) -. "$PSScriptRoot\..\lib\versions.ps1" # 'Select-CurrentVersion' . "$PSScriptRoot\..\lib\manifest.ps1" # 'generate_user_manifest' 'Get-Manifest' -. "$PSScriptRoot\..\lib\download.ps1" +. "$PSScriptRoot\..\lib\install.ps1" if (get_config USE_SQLITE_CACHE) { . "$PSScriptRoot\..\lib\database.ps1" } diff --git a/libexec/scoop-home.ps1 b/libexec/scoop-home.ps1 index b1bd94e356..da495454ad 100644 --- a/libexec/scoop-home.ps1 +++ b/libexec/scoop-home.ps1 @@ -2,9 +2,7 @@ # Summary: Opens the app homepage param($app) -. "$PSScriptRoot\..\lib\versions.ps1" # 'Select-CurrentVersion' . "$PSScriptRoot\..\lib\manifest.ps1" # 'Get-Manifest' -. "$PSScriptRoot\..\lib\download.ps1" # 'Get-UserAgent' if ($app) { $null, $manifest, $bucket, $null = Get-Manifest $app diff --git a/libexec/scoop-info.ps1 b/libexec/scoop-info.ps1 index 1322e33a63..3907b9f71a 100644 --- a/libexec/scoop-info.ps1 +++ b/libexec/scoop-info.ps1 @@ -5,11 +5,9 @@ . "$PSScriptRoot\..\lib\getopt.ps1" . "$PSScriptRoot\..\lib\manifest.ps1" # 'Get-Manifest' -. "$PSScriptRoot\..\lib\versions.ps1" # 'Get-InstalledVersion', 'Select-CurrentVersion' -. "$PSScriptRoot\..\lib\download.ps1" # 'Get-RemoteFileSize' +. "$PSScriptRoot\..\lib\versions.ps1" # 'Get-InstalledVersion' $opt, $app, $err = getopt $args 'v' 'verbose' -$original_app = $app if ($err) { error "scoop info: $err"; exit 1 } $verbose = $opt.v -or $opt.verbose @@ -24,7 +22,7 @@ if (!$manifest) { $global = installed $app $true $status = app_status $app $global $install = install_info $app $status.version $global -$status.installed = ($bucket -and $install.bucket -eq $bucket) -or (installed $app) +$status.installed = $bucket -and $install.bucket -eq $bucket $version_output = $manifest.version $manifest_file = if ($bucket) { manifest_path $app $bucket @@ -32,30 +30,12 @@ $manifest_file = if ($bucket) { $url } -# Standalone and Source detection -if ((Test-Path $original_app) -or ($original_app -match '^(ht|f)tps?://|\\\\')) { - $standalone = $true - if (Test-Path $original_app) { - $original_app = (Get-AbsolutePath "$original_app") - } - if ($install.url) { - if (Test-Path $install.url) { - $install_url = (Get-AbsolutePath $install.url) - } else { - $install_url = $install.url - } - } - if ($original_app -eq $install_url) { - $same_source = $true - } -} - if ($verbose) { $dir = currentdir $app $global $original_dir = versiondir $app $manifest.version $global $persist_dir = persistdir $app $global } else { - $dir, $original_dir, $persist_dir = '', '', '' + $dir, $original_dir, $persist_dir = "", "", "" } if ($status.installed) { @@ -63,39 +43,21 @@ if ($status.installed) { if ($install.url) { $manifest_file = $install.url } - if ($status.deprecated) { - $manifest_file = $status.deprecated - } elseif ($standalone -and !$same_source) { - $version_output = $manifest.version - } elseif ($status.version -eq $manifest.version) { + if ($status.version -eq $manifest.version) { $version_output = $status.version } else { $version_output = "$($status.version) (Update to $($manifest.version) available)" } - } $item = [ordered]@{ Name = $app } -if ($status.deprecated) { - $item.Name += ' (DEPRECATED)' -} if ($manifest.description) { $item.Description = $manifest.description } $item.Version = $version_output - -$item.Source = if ($standalone) { - $original_app -} else { - if ($install.bucket) { - $install.bucket - } elseif ($install.url) { - $install.url - } else { - $bucket - } +if ($bucket) { + $item.Bucket = $bucket } - if ($manifest.homepage) { $item.Website = $manifest.homepage.TrimEnd('/') } @@ -107,7 +69,7 @@ if ($manifest.license) { $manifest.license } elseif ($manifest.license -match '[|,]') { if ($verbose) { - "$($manifest.license) ($(($manifest.license -Split '\||,' | ForEach-Object { "https://spdx.org/licenses/$_.html" }) -join ', '))" + "$($manifest.license) ($(($manifest.license -Split "\||," | ForEach-Object { "https://spdx.org/licenses/$_.html" }) -join ', '))" } else { $manifest.license } @@ -138,13 +100,11 @@ if ($verbose) { $item.Manifest = $manifest_file } if ($status.installed) { # Show installed versions - if (!$standalone -or $same_source) { - $installed_output = @() - Get-InstalledVersion -AppName $app -Global:$global | ForEach-Object { - $installed_output += if ($verbose) { versiondir $app $_ $global } else { "$_$(if ($global) { ' *global*' })" } - } - $item.Installed = $installed_output -join "`n" + $installed_output = @() + Get-InstalledVersion -AppName $app -Global:$global | ForEach-Object { + $installed_output += if ($verbose) { versiondir $app $_ $global } else { "$_$(if ($global) { " *global*" })" } } + $item.Installed = $installed_output -join "`n" if ($verbose) { # Show size of installation @@ -201,12 +161,12 @@ if ($status.installed) { foreach ($url in @(url $manifest (Get-DefaultArchitecture))) { try { if (Test-Path (cache_path $app $manifest.version $url)) { - $cached = ' (latest version is cached)' + $cached = " (latest version is cached)" } else { $cached = $null } - $urlLength = Get-RemoteFileSize $url + $urlLength = (Invoke-WebRequest $url -Method Head).Headers.'Content-Length' | ForEach-Object { [int]$_ } $totalPackage += $urlLength } catch [System.Management.Automation.RuntimeException] { $totalPackage = 0 @@ -236,7 +196,7 @@ if ($binaries) { $binary_output += $_ } } - $item.Binaries = $binary_output -join ' | ' + $item.Binaries = $binary_output -join " | " } $shortcuts = @(arch_specific 'shortcuts' $manifest $install.architecture) if ($shortcuts) { @@ -244,7 +204,7 @@ if ($shortcuts) { $shortcuts | ForEach-Object { $shortcut_output += $_[1] } - $item.Shortcuts = $shortcut_output -join ' | ' + $item.Shortcuts = $shortcut_output -join " | " } $env_set = arch_specific 'env_set' $manifest $install.architecture if ($env_set) { diff --git a/libexec/scoop-install.ps1 b/libexec/scoop-install.ps1 index 0fadcff09d..b7dda05d3c 100644 --- a/libexec/scoop-install.ps1 +++ b/libexec/scoop-install.ps1 @@ -33,7 +33,6 @@ . "$PSScriptRoot\..\lib\manifest.ps1" # 'generate_user_manifest' 'Get-Manifest' 'Select-CurrentVersion' (indirectly) . "$PSScriptRoot\..\lib\system.ps1" . "$PSScriptRoot\..\lib\install.ps1" -. "$PSScriptRoot\..\lib\download.ps1" . "$PSScriptRoot\..\lib\decompress.ps1" . "$PSScriptRoot\..\lib\shortcuts.ps1" . "$PSScriptRoot\..\lib\psmodules.ps1" diff --git a/libexec/scoop-list.ps1 b/libexec/scoop-list.ps1 index f757146d47..da44dd1410 100644 --- a/libexec/scoop-list.ps1 +++ b/libexec/scoop-list.ps1 @@ -5,9 +5,8 @@ param($query) . "$PSScriptRoot\..\lib\versions.ps1" # 'Select-CurrentVersion' . "$PSScriptRoot\..\lib\manifest.ps1" # 'parse_json' 'Select-CurrentVersion' (indirectly) -. "$PSScriptRoot\..\lib\download.ps1" # 'Get-UserAgent' -$defaultArchitecture = Get-DefaultArchitecture +$def_arch = Get-DefaultArchitecture if (-not (Get-FormatData ScoopApps)) { Update-FormatData "$PSScriptRoot\..\supporting\formats\ScoopTypes.Format.ps1xml" } @@ -48,11 +47,10 @@ $apps | Where-Object { !$query -or ($_.name -match $query) } | ForEach-Object { $item.Updated = $updated $info = @() - if ((app_status $app $global).deprecated) { $info += 'Deprecated package'} if ($global) { $info += 'Global install' } if (failed $app $global) { $info += 'Install failed' } if ($install_info.hold) { $info += 'Held package' } - if ($install_info.architecture -and $defaultArchitecture -ne $install_info.architecture) { + if ($install_info.architecture -and $def_arch -ne $install_info.architecture) { $info += $install_info.architecture } $item.Info = $info -join ', ' diff --git a/libexec/scoop-search.ps1 b/libexec/scoop-search.ps1 index d94b18660d..4254099989 100644 --- a/libexec/scoop-search.ps1 +++ b/libexec/scoop-search.ps1 @@ -10,10 +10,15 @@ param($query) . "$PSScriptRoot\..\lib\manifest.ps1" # 'manifest' . "$PSScriptRoot\..\lib\versions.ps1" # 'Get-LatestVersion' -. "$PSScriptRoot\..\lib\download.ps1" $list = [System.Collections.Generic.List[PSCustomObject]]::new() +$githubtoken = Get-GitHubToken +$authheader = @{} +if ($githubtoken) { + $authheader = @{'Authorization' = "token $githubtoken" } +} + function bin_match($manifest, $query) { if (!$manifest.bin) { return $false } $bins = foreach ($bin in $manifest.bin) { @@ -117,6 +122,23 @@ function search_bucket_legacy($bucket, $query) { } } +function download_json($url) { + $ProgressPreference = 'SilentlyContinue' + $result = Invoke-WebRequest $url -UseBasicParsing -Headers $authheader | Select-Object -ExpandProperty content | ConvertFrom-Json + $ProgressPreference = 'Continue' + $result +} + +function github_ratelimit_reached { + $api_link = 'https://api.github.com/rate_limit' + $ret = (download_json $api_link).rate.remaining -eq 0 + if ($ret) { + Write-Host "GitHub API rate limit reached. +Please try again later or configure your API token using 'scoop config gh_token '." + } + $ret +} + function search_remote($bucket, $query) { $uri = [System.Uri](known_bucket_repo $bucket) if ($uri.AbsolutePath -match '/([a-zA-Z0-9]*)/([a-zA-Z0-9-]*)(?:.git|/)?') { diff --git a/libexec/scoop-status.ps1 b/libexec/scoop-status.ps1 index cc34ddac9f..a62cad2dd1 100644 --- a/libexec/scoop-status.ps1 +++ b/libexec/scoop-status.ps1 @@ -6,7 +6,6 @@ . "$PSScriptRoot\..\lib\manifest.ps1" # 'manifest' 'parse_json' "install_info" . "$PSScriptRoot\..\lib\versions.ps1" # 'Select-CurrentVersion' -. "$PSScriptRoot\..\lib\download.ps1" # 'Get-UserAgent' # check if scoop needs updating $currentdir = versiondir 'scoop' 'current' @@ -59,7 +58,7 @@ $true, $false | ForEach-Object { # local and global apps Get-ChildItem $dir | Where-Object name -NE 'scoop' | ForEach-Object { $app = $_.name $status = app_status $app $global - if (!$status.outdated -and !$status.failed -and !$status.deprecated -and !$status.removed -and !$status.missing_deps) { return } + if (!$status.outdated -and !$status.failed -and !$status.removed -and !$status.missing_deps) { return } $item = [ordered]@{} $item.Name = $app @@ -67,10 +66,9 @@ $true, $false | ForEach-Object { # local and global apps $item.'Latest Version' = if ($status.outdated) { $status.latest_version } else { "" } $item.'Missing Dependencies' = $status.missing_deps -Split ' ' -Join ' | ' $info = @() - if ($status.failed) { $info += 'Install failed' } - if ($status.hold) { $info += 'Held package' } - if ($status.deprecated) { $info += 'Deprecated' } - if ($status.removed) { $info += 'Manifest removed' } + if ($status.failed) { $info += 'Install failed' } + if ($status.hold) { $info += 'Held package' } + if ($status.removed) { $info += 'Manifest removed' } $item.Info = $info -join ', ' $list += [PSCustomObject]$item } diff --git a/libexec/scoop-uninstall.ps1 b/libexec/scoop-uninstall.ps1 index 5bdd57e5d1..5ad606a461 100644 --- a/libexec/scoop-uninstall.ps1 +++ b/libexec/scoop-uninstall.ps1 @@ -74,7 +74,7 @@ if (!$apps) { exit 0 } continue } - Invoke-Installer -Path $dir -Manifest $manifest -ProcessorArchitecture $architecture -Global $global -Uninstall + Invoke-Installer -Path $dir -Manifest $manifest -ProcessorArchitecture $architecture -Uninstall rm_shims $app $manifest $global $architecture rm_startmenu_shortcuts $manifest $global $architecture diff --git a/libexec/scoop-update.ps1 b/libexec/scoop-update.ps1 index 9d41906eca..bd825731c3 100644 --- a/libexec/scoop-update.ps1 +++ b/libexec/scoop-update.ps1 @@ -24,7 +24,6 @@ . "$PSScriptRoot\..\lib\versions.ps1" . "$PSScriptRoot\..\lib\depends.ps1" . "$PSScriptRoot\..\lib\install.ps1" -. "$PSScriptRoot\..\lib\download.ps1" if (get_config USE_SQLITE_CACHE) { . "$PSScriptRoot\..\lib\database.ps1" } diff --git a/libexec/scoop-virustotal.ps1 b/libexec/scoop-virustotal.ps1 index 8fe63359cf..0782c7e225 100644 --- a/libexec/scoop-virustotal.ps1 +++ b/libexec/scoop-virustotal.ps1 @@ -29,10 +29,9 @@ # -p, --passthru Return reports as objects . "$PSScriptRoot\..\lib\getopt.ps1" -. "$PSScriptRoot\..\lib\versions.ps1" # 'Select-CurrentVersion' . "$PSScriptRoot\..\lib\manifest.ps1" # 'Get-Manifest' . "$PSScriptRoot\..\lib\json.ps1" # 'json_path' -. "$PSScriptRoot\..\lib\download.ps1" # 'hash_for_url' +. "$PSScriptRoot\..\lib\install.ps1" # 'hash_for_url' . "$PSScriptRoot\..\lib\depends.ps1" # 'Get-Dependency' $opt, $apps, $err = getopt $args 'asnup' @('all', 'scan', 'no-depends', 'no-update-scoop', 'passthru') @@ -87,6 +86,11 @@ Function ConvertTo-VirusTotalUrlId ($url) { $url_id } +Function Get-RemoteFileSize ($url) { + $response = Invoke-WebRequest -Uri $url -Method HEAD -UseBasicParsing + $response.Headers.'Content-Length' | ForEach-Object { [System.Convert]::ToInt32($_) } +} + Function Get-VirusTotalResultByHash ($hash, $url, $app) { $hash = $hash.ToLower() $api_url = "https://www.virustotal.com/api/v3/files/$hash" diff --git a/supporting/shims/kiennq/checksum.sha256 b/supporting/shims/kiennq/checksum.sha256 index 527e378ce5..51fe5eeaf7 100644 --- a/supporting/shims/kiennq/checksum.sha256 +++ b/supporting/shims/kiennq/checksum.sha256 @@ -1 +1 @@ -140e3801d8adeda639a21b14e62b93a4c7d26b7a758421f43c82be59753be49b *shim.exe +410f84fe347cf55f92861ea3899d30b2d84a8bbc56bb3451d74697a4a0610b25 *shim.exe diff --git a/supporting/shims/kiennq/checksum.sha512 b/supporting/shims/kiennq/checksum.sha512 index 18d66978c3..6c39fc8386 100644 --- a/supporting/shims/kiennq/checksum.sha512 +++ b/supporting/shims/kiennq/checksum.sha512 @@ -1 +1 @@ -59d9da9f9714003b915bcafbe1b41f53b121dde206ecc23984f62273e957766eece8d64ffc53011c328d3a2ad627aa0f4f7c39bbec8e7b64d0d2ee7b7e771423 *shim.exe +9ce94adf48f7a31ab5773465582728c39db6f11a560fc43316fe6c1ad0a7b69a76aa3f9b52bb6b2e3be8043e4920985c8ca0bf157be9bf1e4a5a4d7c4ed195ba *shim.exe diff --git a/supporting/shims/kiennq/shim.exe b/supporting/shims/kiennq/shim.exe index bb12009124..3ab79dd5e6 100644 Binary files a/supporting/shims/kiennq/shim.exe and b/supporting/shims/kiennq/shim.exe differ diff --git a/supporting/shims/kiennq/version.txt b/supporting/shims/kiennq/version.txt index d95827c3d9..903cd9f2a0 100644 --- a/supporting/shims/kiennq/version.txt +++ b/supporting/shims/kiennq/version.txt @@ -1 +1 @@ -v3.1.2 +v3.1.1 diff --git a/test/Scoop-Core.Tests.ps1 b/test/Scoop-Core.Tests.ps1 index d1cc9defe3..f6846e5456 100644 --- a/test/Scoop-Core.Tests.ps1 +++ b/test/Scoop-Core.Tests.ps1 @@ -73,6 +73,28 @@ Describe 'Test-HelperInstalled' -Tag 'Scoop' { } } +Describe 'Test-Aria2Enabled' -Tag 'Scoop' { + It 'should return true if aria2 is installed' { + Mock Test-HelperInstalled { $true } + Mock get_config { $true } + Test-Aria2Enabled | Should -BeTrue + } + + It 'should return false if aria2 is not installed' { + Mock Test-HelperInstalled { $false } + Mock get_config { $false } + Test-Aria2Enabled | Should -BeFalse + + Mock Test-HelperInstalled { $false } + Mock get_config { $true } + Test-Aria2Enabled | Should -BeFalse + + Mock Test-HelperInstalled { $true } + Mock get_config { $false } + Test-Aria2Enabled | Should -BeFalse + } +} + Describe 'Test-CommandAvailable' -Tag 'Scoop' { It 'should return true if command exists' { Test-CommandAvailable 'Write-Host' | Should -BeTrue diff --git a/test/Scoop-Decompress.Tests.ps1 b/test/Scoop-Decompress.Tests.ps1 index e635492cec..4ec31a4ece 100644 --- a/test/Scoop-Decompress.Tests.ps1 +++ b/test/Scoop-Decompress.Tests.ps1 @@ -25,7 +25,7 @@ Describe 'Decompression function' -Tag 'Scoop', 'Windows', 'Decompress' { } It 'Test cases should exist and hash should match' { $testcases | Should -Exist - (Get-FileHash -Path $testcases -Algorithm SHA256).Hash.ToLower() | Should -Be '591072faabd419b77932b7023e5899b4e05c0bf8e6859ad367398e6bfe1eb203' + (Get-FileHash -Path $testcases -Algorithm SHA256).Hash.ToLower() | Should -Be 'afb86b0552187b8d630ce25d02835fb809af81c584f07e54cb049fb74ca134b6' } It 'Test cases should be extracted correctly' { { Microsoft.PowerShell.Archive\Expand-Archive -Path $testcases -DestinationPath $working_dir } | Should -Not -Throw @@ -61,7 +61,7 @@ Describe 'Decompression function' -Tag 'Scoop', 'Windows', 'Decompress' { $to = test_extract 'Expand-7zipArchive' $test1 $to | Should -Exist "$to\empty" | Should -Exist - (Get-ChildItem $to).Count | Should -Be 4 + (Get-ChildItem $to).Count | Should -Be 3 } It 'extract "extract_dir" correctly' { @@ -78,14 +78,6 @@ Describe 'Decompression function' -Tag 'Scoop', 'Windows', 'Decompress' { (Get-ChildItem $to).Count | Should -Be 1 } - It 'extract "extract_dir" with nested folder with same name' { - $to = test_extract 'Expand-7zipArchive' $test1 $false 'keep\sub' - $to | Should -Exist - "$to\keep\empty" | Should -Exist - (Get-ChildItem $to).Count | Should -Be 1 - (Get-ChildItem "$to\keep").Count | Should -Be 1 - } - It 'extract nested compressed file' { # file ext: tgz $to = test_extract 'Expand-7zipArchive' $test2 diff --git a/test/Scoop-Depends.Tests.ps1 b/test/Scoop-Depends.Tests.ps1 index 3e6713d050..79b868ef90 100644 --- a/test/Scoop-Depends.Tests.ps1 +++ b/test/Scoop-Depends.Tests.ps1 @@ -65,7 +65,6 @@ Describe 'Package Dependencies' -Tag 'Scoop' { BeforeAll { Mock Test-HelperInstalled { $false } Mock get_config { $true } -ParameterFilter { $name -eq 'USE_LESSMSI' } - Mock get_config { $false } -ParameterFilter { $name -eq 'USE_EXTERNAL_7ZIP' } Mock Get-Manifest { 'lessmsi', @{}, $null, $null } -ParameterFilter { $app -eq 'lessmsi' } Mock Get-Manifest { '7zip', @{ url = 'test.msi' }, $null, $null } -ParameterFilter { $app -eq '7zip' } Mock Get-Manifest { 'innounp', @{}, $null, $null } -ParameterFilter { $app -eq 'innounp' } diff --git a/test/Scoop-Download.Tests.ps1 b/test/Scoop-Download.Tests.ps1 deleted file mode 100644 index 8968c30014..0000000000 --- a/test/Scoop-Download.Tests.ps1 +++ /dev/null @@ -1,49 +0,0 @@ -BeforeAll { - . "$PSScriptRoot\Scoop-TestLib.ps1" - . "$PSScriptRoot\..\lib\core.ps1" - . "$PSScriptRoot\..\lib\download.ps1" -} - -Describe 'Test-Aria2Enabled' -Tag 'Scoop' { - It 'should return true if aria2 is installed' { - Mock Test-HelperInstalled { $true } - Mock get_config { $true } - Test-Aria2Enabled | Should -BeTrue - } - - It 'should return false if aria2 is not installed' { - Mock Test-HelperInstalled { $false } - Mock get_config { $false } - Test-Aria2Enabled | Should -BeFalse - - Mock Test-HelperInstalled { $false } - Mock get_config { $true } - Test-Aria2Enabled | Should -BeFalse - - Mock Test-HelperInstalled { $true } - Mock get_config { $false } - Test-Aria2Enabled | Should -BeFalse - } -} - -Describe 'url_filename' -Tag 'Scoop' { - It 'should extract the real filename from an url' { - url_filename 'http://example.org/foo.txt' | Should -Be 'foo.txt' - url_filename 'http://example.org/foo.txt?var=123' | Should -Be 'foo.txt' - } - - It 'can be tricked with a hash to override the real filename' { - url_filename 'http://example.org/foo-v2.zip#/foo.zip' | Should -Be 'foo.zip' - } -} - -Describe 'url_remote_filename' -Tag 'Scoop' { - It 'should extract the real filename from an url' { - url_remote_filename 'http://example.org/foo.txt' | Should -Be 'foo.txt' - url_remote_filename 'http://example.org/foo.txt?var=123' | Should -Be 'foo.txt' - } - - It 'can not be tricked with a hash to override the real filename' { - url_remote_filename 'http://example.org/foo-v2.zip#/foo.zip' | Should -Be 'foo-v2.zip' - } -} diff --git a/test/Scoop-Install.Tests.ps1 b/test/Scoop-Install.Tests.ps1 index 3e564d1964..966c2c0c50 100644 --- a/test/Scoop-Install.Tests.ps1 +++ b/test/Scoop-Install.Tests.ps1 @@ -12,6 +12,28 @@ Describe 'appname_from_url' -Tag 'Scoop' { } } +Describe 'url_filename' -Tag 'Scoop' { + It 'should extract the real filename from an url' { + url_filename 'http://example.org/foo.txt' | Should -Be 'foo.txt' + url_filename 'http://example.org/foo.txt?var=123' | Should -Be 'foo.txt' + } + + It 'can be tricked with a hash to override the real filename' { + url_filename 'http://example.org/foo-v2.zip#/foo.zip' | Should -Be 'foo.zip' + } +} + +Describe 'url_remote_filename' -Tag 'Scoop' { + It 'should extract the real filename from an url' { + url_remote_filename 'http://example.org/foo.txt' | Should -Be 'foo.txt' + url_remote_filename 'http://example.org/foo.txt?var=123' | Should -Be 'foo.txt' + } + + It 'can not be tricked with a hash to override the real filename' { + url_remote_filename 'http://example.org/foo-v2.zip#/foo.zip' | Should -Be 'foo-v2.zip' + } +} + Describe 'is_in_dir' -Tag 'Scoop', 'Windows' { It 'should work correctly' { is_in_dir 'C:\test' 'C:\foo' | Should -BeFalse diff --git a/test/fixtures/decompress/TestCases.zip b/test/fixtures/decompress/TestCases.zip index fb2092e56c..d2cd98c0f2 100644 Binary files a/test/fixtures/decompress/TestCases.zip and b/test/fixtures/decompress/TestCases.zip differ