diff --git a/internal/backend/builtin.go b/internal/backend/builtin.go index 20c6953c..0ac6b8e5 100644 --- a/internal/backend/builtin.go +++ b/internal/backend/builtin.go @@ -54,9 +54,29 @@ func runBuiltin(ctx context.Context, invocation *builderInvocation) error { func fetchURL(ctx context.Context, drv *zbstore.Derivation, realStoreDir string) error { href := drv.Env["url"] - if href == "" { + var urls []string + if href != "" { + urls = append(urls, href) + } + + hrefs := drv.Env["urls"] + urls = append(urls, strings.Fields(hrefs)...) + if len(urls) == 0 { return fmt.Errorf("missing url environment variable") } + + var err error + for _, href := range urls { + err = getURL(ctx, drv, realStoreDir, href) + if err == nil { + break + } + } + + return err +} + +func getURL(ctx context.Context, drv *zbstore.Derivation, realStoreDir string, href string) error { outputPath := drv.Env[zbstore.DefaultDerivationOutputName] if outputPath == "" { return fmt.Errorf("missing %s environment variable", zbstore.DefaultDerivationOutputName) diff --git a/internal/backend/realize_test.go b/internal/backend/realize_test.go index f62a562d..69eb6b42 100644 --- a/internal/backend/realize_test.go +++ b/internal/backend/realize_test.go @@ -1150,76 +1150,143 @@ func TestRealizeCores(t *testing.T) { } func TestRealizeFetchURL(t *testing.T) { - ctx := testcontext.New(t) - dir := backendtest.NewStoreDirectory(t) - - const fileContent = "Hello, World!\n" - mux := http.NewServeMux() - mux.HandleFunc("/hello.txt", func(w http.ResponseWriter, r *http.Request) { - http.ServeContent(w, r, "hello.txt", time.Time{}, strings.NewReader(fileContent)) - }) - srv := httptest.NewServer(mux) - defer srv.Close() - - exportBuffer := new(bytes.Buffer) - exporter := zbstore.NewExportWriter(exportBuffer) - const wantOutputName = "hello.txt" - wantOutputCA := nix.FlatFileContentAddress(mustParseHash(t, "sha256:c98c24b677eff44860afea6f493bbaec5bb1c4cbb209c6fc2bbb47f66ff2ad31")) - drvContent := &zbstore.Derivation{ - Name: wantOutputName, - Dir: dir, - Builder: "builtin:fetchurl", - System: "builtin", - Env: map[string]string{ - "url": string(srv.URL + "/hello.txt"), - "out": zbstore.HashPlaceholder("out"), + tests := []struct { + name string + env func(string) map[string]string + wantOutputCA nix.ContentAddress + wantOutputName string + fileContent string + expectBuildFail bool + }{ + { + name: "FetchSingularUrl", + env: func(urlBase string) map[string]string { + return map[string]string{ + "url": urlBase + "/hello.txt", + "out": zbstore.HashPlaceholder("out"), + } + }, + wantOutputCA: nix.FlatFileContentAddress(mustParseHash(t, "sha256:c98c24b677eff44860afea6f493bbaec5bb1c4cbb209c6fc2bbb47f66ff2ad31")), + wantOutputName: "hello.txt", + fileContent: "Hello, World!\n", }, - Outputs: map[string]*zbstore.DerivationOutputType{ - zbstore.DefaultDerivationOutputName: zbstore.FixedCAOutput(wantOutputCA), + { + name: "FetchBrokenUrls", + env: func(urlBase string) map[string]string { + return map[string]string{ + "urls": "http://broken/hello.txt", + "out": zbstore.HashPlaceholder("out"), + } + }, + wantOutputCA: nix.FlatFileContentAddress(mustParseHash(t, "sha256:c98c24b677eff44860afea6f493bbaec5bb1c4cbb209c6fc2bbb47f66ff2ad31")), + wantOutputName: "hello.txt", + fileContent: "Hello, World!\n", + expectBuildFail: true, + }, + { + name: "FetchNoUrl", + env: func(urlBase string) map[string]string { + return map[string]string{} + }, + wantOutputCA: nix.FlatFileContentAddress(mustParseHash(t, "sha256:c98c24b677eff44860afea6f493bbaec5bb1c4cbb209c6fc2bbb47f66ff2ad31")), + wantOutputName: "hello.txt", + fileContent: "Hello, World!\n", + expectBuildFail: true, + }, + { + name: "FetchFromUrlsList", + env: func(urlBase string) map[string]string { + return map[string]string{ + "urls": fmt.Sprintf("%s, %s", urlBase+"/bad/hello.txt", urlBase+"/hello.txt"), + "out": zbstore.HashPlaceholder("out"), + } + }, + wantOutputCA: nix.FlatFileContentAddress(mustParseHash(t, "sha256:c98c24b677eff44860afea6f493bbaec5bb1c4cbb209c6fc2bbb47f66ff2ad31")), + wantOutputName: "hello.txt", + fileContent: "Hello, World!\n", }, - } - drvPath, _, err := storetest.ExportDerivation(exporter, drvContent) - if err != nil { - t.Fatal(err) - } - if err := exporter.Close(); err != nil { - t.Fatal(err) } - _, client, err := backendtest.NewServer(ctx, t, dir, &backendtest.Options{ - TempDir: t.TempDir(), - }) - if err != nil { - t.Fatal(err) - } - codec, releaseCodec, err := storeCodec(ctx, client) - if err != nil { - t.Fatal(err) - } - err = codec.Export(nil, exportBuffer) - releaseCodec() - if err != nil { - t.Fatal(err) - } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + ctx := testcontext.New(t) + dir := backendtest.NewStoreDirectory(t) - realizeResponse := new(zbstorerpc.RealizeResponse) - err = jsonrpc.Do(ctx, client, zbstorerpc.RealizeMethod, realizeResponse, &zbstorerpc.RealizeRequest{ - DrvPaths: []zbstore.Path{drvPath}, - }) - if err != nil { - t.Fatal("build drv:", err) - } - got, err := backendtest.WaitForSuccessfulBuild(ctx, client, realizeResponse.BuildID) - if err != nil { - gotLog, _ := backendtest.ReadLog(ctx, client, realizeResponse.BuildID, drvPath) - t.Fatalf("build drv: %v\nlog:\n%s", err, gotLog) - } + mux := http.NewServeMux() + mux.HandleFunc( + "/hello.txt", + func(w http.ResponseWriter, r *http.Request) { + http.ServeContent( + w, + r, + "hello.txt", + time.Time{}, + strings.NewReader(test.fileContent)) + }) + + srv := httptest.NewServer(mux) + defer srv.Close() - wantOutputPath, err := zbstore.FixedCAOutputPath(dir, wantOutputName, wantOutputCA, zbstore.References{}) - if err != nil { - t.Fatal(err) + exportBuffer := new(bytes.Buffer) + exporter := zbstore.NewExportWriter(exportBuffer) + drvContent := &zbstore.Derivation{ + Name: test.wantOutputName, + Dir: dir, + Builder: "builtin:fetchurl", + System: "builtin", + Env: test.env(srv.URL), + Outputs: map[string]*zbstore.DerivationOutputType{ + zbstore.DefaultDerivationOutputName: zbstore.FixedCAOutput(test.wantOutputCA), + }, + } + drvPath, _, err := storetest.ExportDerivation(exporter, drvContent) + if err != nil { + t.Fatal(err) + } + if err := exporter.Close(); err != nil { + t.Fatal(err) + } + + _, client, err := backendtest.NewServer(ctx, t, dir, &backendtest.Options{ + TempDir: t.TempDir(), + }) + if err != nil { + t.Fatal(err) + } + codec, releaseCodec, err := storeCodec(ctx, client) + if err != nil { + t.Fatal(err) + } + err = codec.Export(nil, exportBuffer) + releaseCodec() + if err != nil { + t.Fatal(err) + } + + realizeResponse := new(zbstorerpc.RealizeResponse) + err = jsonrpc.Do(ctx, client, zbstorerpc.RealizeMethod, realizeResponse, &zbstorerpc.RealizeRequest{ + DrvPaths: []zbstore.Path{drvPath}, + }) + if err != nil { + t.Fatal("build drv:", err) + } + // expect build to fail + got, err := backendtest.WaitForSuccessfulBuild(ctx, client, realizeResponse.BuildID) + if err != nil && test.expectBuildFail { + return + } + if err != nil { + gotLog, _ := backendtest.ReadLog(ctx, client, realizeResponse.BuildID, drvPath) + t.Fatalf("build drv: %v\nlog:\n%s", err, gotLog) + } + + wantOutputPath, err := zbstore.FixedCAOutputPath(dir, test.wantOutputName, test.wantOutputCA, zbstore.References{}) + if err != nil { + t.Fatal(err) + } + checkSingleFileOutput(t, drvPath, wantOutputPath, []byte(test.fileContent), got) + }) } - checkSingleFileOutput(t, drvPath, wantOutputPath, []byte(fileContent), got) } func TestRealizeSignature(t *testing.T) { diff --git a/internal/frontend/prelude.lua b/internal/frontend/prelude.lua index cc9b33f6..268597cc 100644 --- a/internal/frontend/prelude.lua +++ b/internal/frontend/prelude.lua @@ -15,10 +15,20 @@ local function baseNameOf(path) return base end ----@param args {url: string, hash: string, name: string?, executable: boolean?} +---@param args {hash: string, name: string?, executable: boolean?, url: string?, urls: string[]?} ---@return derivation function fetchurl(args) - local name = args.name or baseNameOf(args.url) + if (args.url == nil or args.url == "") and (args.urls == nil or next(args.urls) == nil) then + error("Either url or urls must be set") + end + + local name + if args.url == nil then + name = args.name or baseNameOf(args.urls[1]) + else + name = args.name or baseNameOf(args.url) + end + local outputHashMode = "flat" if args.executable then outputHashMode = "recursive" @@ -29,7 +39,7 @@ function fetchurl(args) system = "builtin"; url = args.url; - urls = { args.url }; + urls = args.urls; executable = args.executable or false; unpack = false; outputHash = args.hash; diff --git a/internal/frontend/prelude.luac b/internal/frontend/prelude.luac index c042c5b3..25740817 100644 Binary files a/internal/frontend/prelude.luac and b/internal/frontend/prelude.luac differ diff --git a/zb_defs.lua b/zb_defs.lua index 37fd4abf..fa24c11a 100644 --- a/zb_defs.lua +++ b/zb_defs.lua @@ -50,7 +50,7 @@ function storePath(path) end function toFile(name, s) end ---Create a derivation that downloads a URL. ----@param args {url: string, hash: string, name: string?, executable: boolean?} +---@param args {hash: string, name: string?, executable: boolean?, url: string?, urls: string[]?} ---@return derivation function fetchurl(args) end