diff --git a/docs/pages.jl b/docs/pages.jl index 1da2f2e8..54e9d759 100644 --- a/docs/pages.jl +++ b/docs/pages.jl @@ -4,22 +4,22 @@ pages = [ "Basics" => "tutorials.md", "Radials" => "radials.md", "Kriging" => "kriging.md", - "Gaussian Process" => "abstractgps.md", + "Gaussian process" => "abstractgps.md", "Lobachevsky" => "lobachevsky.md", "Linear" => "LinearSurrogate.md", - "InverseDistance" => "InverseDistance.md", + "Inverse distance" => "InverseDistance.md", "XGBoost" => "xgboost.md", - "SecondOrderPolynomial" => "secondorderpoly.md", + "Second order polynomial" => "secondorderpoly.md", "Earth (MARS)" => "earth.md", - "NeuralSurrogate" => "neural.md", + "Neural network" => "neural.md", "Wendland" => "wendland.md", - "Polynomial Chaos" => "polychaos.md", - "Variable Fidelity" => "variablefidelity.md", - "Gradient Enhanced Kriging" => "gek.md", + "Polynomial chaos" => "polychaos.md", + "Variable fidelity" => "variablefidelity.md", + "Gradient enhanced Kriging" => "gek.md", "GEKPLS" => "gekpls.md", "KPLS and KPLSK" => "kpls.md", "MOE" => "moe.md", - "Parallel Optimization" => "parallel.md", + "Parallel optimization" => "parallel.md", ] "User guide" => [ "Samples" => "samples.md", @@ -32,14 +32,14 @@ pages = [ "Lp norm" => "lp.md", "Rosenbrock" => "rosenbrock.md", "Tensor product" => "tensor_prod.md", - "Cantilever beam" => "cantilever.md", - "Water Flow function" => "water_flow.md", + "Cantilever beam function" => "cantilever.md", + "Water flow function" => "water_flow.md", "Welded beam function" => "welded_beam.md", "Branin function" => "BraninFunction.md", "Improved Branin function" => "ImprovedBraninFunction.md", "Ackley function" => "ackley.md", - "Gramacy & Lee Function" => "gramacylee.md", - "Salustowicz Benchmark" => "Salustowicz.md", - "Multi objective optimization" => "multi_objective_opt.md", + "Gramacy & Lee function" => "gramacylee.md", + "Salustowicz benchmark" => "Salustowicz.md", + "Multi-objective optimization" => "multi_objective_opt.md", ] ] diff --git a/docs/src/BraninFunction.md b/docs/src/BraninFunction.md index 190796fc..794b2b2e 100644 --- a/docs/src/BraninFunction.md +++ b/docs/src/BraninFunction.md @@ -40,12 +40,13 @@ lower_bound = [-5, 0] upper_bound = [10, 15] xys = sample(n_samples, lower_bound, upper_bound, SobolSample()) zs = branin.(xys) -x, y = -5.00:10.00, 0.00:15.00 -p1 = surface(x, y, (x1, x2) -> branin((x1, x2))) +xgrid = range(lower_bound[1], upper_bound[1], length = 100) +ygrid = range(lower_bound[2], upper_bound[2], length = 100) +p1 = surface(xgrid, ygrid, (x1, x2) -> branin((x1, x2))) xs = [xy[1] for xy in xys] ys = [xy[2] for xy in xys] scatter!(xs, ys, zs) -p2 = contour(x, y, (x1, x2) -> branin((x1, x2))) +p2 = contour(xgrid, ygrid, (x1, x2) -> branin((x1, x2))) scatter!(xs, ys) plot(p1, p2, title = "True function") ``` @@ -58,9 +59,9 @@ radial_surrogate = RadialBasis(xys, zs, lower_bound, upper_bound) ``` ```@example BraninFunction -p1 = surface(x, y, (x, y) -> radial_surrogate([x y])) +p1 = surface(xgrid, ygrid, (x, y) -> radial_surrogate([x y])) scatter!(xs, ys, zs, marker_z = zs) -p2 = contour(x, y, (x, y) -> radial_surrogate([x y])) +p2 = contour(xgrid, ygrid, (x, y) -> radial_surrogate([x y])) scatter!(xs, ys, marker_z = zs) plot(p1, p2, title = "Radial Surrogate") ``` @@ -72,9 +73,9 @@ InverseDistance = InverseDistanceSurrogate(xys, zs, lower_bound, upper_bound) ``` ```@example BraninFunction -p1 = surface(x, y, (x, y) -> InverseDistance([x y])) +p1 = surface(xgrid, ygrid, (x, y) -> InverseDistance([x y])) scatter!(xs, ys, zs, marker_z = zs) -p2 = contour(x, y, (x, y) -> InverseDistance([x y])) +p2 = contour(xgrid, ygrid, (x, y) -> InverseDistance([x y])) scatter!(xs, ys, marker_z = zs) plot(p1, p2, title = "Inverse Distance Surrogate") ``` @@ -87,9 +88,9 @@ Lobachevsky = LobachevskySurrogate( ``` ```@example BraninFunction -p1 = surface(x, y, (x, y) -> Lobachevsky([x y])) +p1 = surface(xgrid, ygrid, (x, y) -> Lobachevsky([x y])) scatter!(xs, ys, zs, marker_z = zs) -p2 = contour(x, y, (x, y) -> Lobachevsky([x y])) +p2 = contour(xgrid, ygrid, (x, y) -> Lobachevsky([x y])) scatter!(xs, ys, marker_z = zs) plot(p1, p2, title = "Lobachevsky Surrogate") ``` diff --git a/docs/src/ImprovedBraninFunction.md b/docs/src/ImprovedBraninFunction.md index 2c253ba4..0e2dbb9e 100644 --- a/docs/src/ImprovedBraninFunction.md +++ b/docs/src/ImprovedBraninFunction.md @@ -1,12 +1,26 @@ -# Branin Function +# Improved Branin function -The Branin Function is commonly used as a test function for metamodelling in computer experiments, especially in the context of optimization. +The [Branin function](BraninFunction.md) is smooth, deterministic, and has three +global minima of equal value. That makes it a clean test, but an unrealistic one: +a real objective evaluated by simulation is rarely reproducible to the last bit. -## Modifications for Improved Branin Function: +The *improved* Branin function adds a time-varying perturbation inside the cosine +term, turning the landscape into something closer to what an optimizer meets in +practice: -To enhance the Branin function, changes were made to introduce irregularities, variability, and a dynamic aspect to its landscape. Here's an example: +```math +f(x_1, x_2, \tau) = \left(x_2 - \frac{5.1}{4\pi^2}x_1^2 + \frac{5}{\pi}x_1 - 6\right)^2 + + 10\left(1 - \frac{1}{8\pi}\right)\cos(x_1 + \tau\,\xi) + 10, +\qquad \xi \sim \mathcal{N}(0, 1) +``` + +where `time_step` is ``\tau``. Because ``\xi`` is drawn afresh on every call, the +objective is **stochastic**: two evaluations at the same point disagree. ```@example improved_branin +using Surrogates, Plots, Random +Random.seed!(42) + function improved_branin(x, time_step) x1 = x[1] x2 = x[2] @@ -17,35 +31,46 @@ function improved_branin(x, time_step) s = 10 t = 1 / (8 * pi) - # Adding noise to the function's output - noise = randn() * time_step # Simulating time-varying noise + # Time-varying perturbation of the phase, redrawn on every call. + noise = randn() * time_step term1 = a * (x2 - b * x1^2 + c * x1 - r)^2 - term2 = s * (1 - t) * cos(x1 + noise) # Introducing dynamic component - y = term1 + term2 + s + term2 = s * (1 - t) * cos(x1 + noise) + return term1 + term2 + s end ``` -This improved function now incorporates irregularities, variability, and a dynamic aspect. These changes aim to make the optimization landscape more challenging and realistic. +The seed matters here: without it, every run of this page — and every build of +these docs — produces a different surface. -## Using the Improved Branin Function: +```@example improved_branin +p = [2.5, 7.5] +repeats = [improved_branin(p, 0.1) for _ in 1:6] +extrema(repeats) +``` -After defining the improved Branin function, you can proceed to test different surrogates and visualize their performance using the updated function. Here's an example of using the improved function with the Radial Basis surrogate: +## Fitting a surrogate ```@example improved_branin -using Surrogates, Plots - n_samples = 80 -lower_bound = [-5, 0] -upper_bound = [10, 15] +lower_bound = [-5.0, 0.0] +upper_bound = [10.0, 15.0] xys = sample(n_samples, lower_bound, upper_bound, SobolSample()) zs = [improved_branin(xy, 0.1) for xy in xys] -radial_surrogate = RadialBasis(xys, zs, lower_bound, upper_bound) -x, y = -5.00:10.00, 0.00:15.00 +kriging_surrogate = Kriging(xys, zs, lower_bound, upper_bound) +``` + +```@example improved_branin +xgrid = range(lower_bound[1], upper_bound[1], length = 100) +ygrid = range(lower_bound[2], upper_bound[2], length = 100) xs = [xy[1] for xy in xys] ys = [xy[2] for xy in xys] -p1 = surface(x, y, (x, y) -> radial_surrogate([x, y])) +p1 = surface(xgrid, ygrid, (x, y) -> kriging_surrogate((x, y))) +scatter!(xs, ys, zs, marker_z = zs) +p2 = contour(xgrid, ygrid, (x, y) -> kriging_surrogate((x, y))) scatter!(xs, ys, marker_z = zs) -p2 = contour(x, y, (x, y) -> radial_surrogate([x, y])) -scatter!(xs, ys, marker_z = zs) -plot(p1, p2, title = "Radial Surrogate") +plot(p1, p2, title = "Kriging surrogate") ``` + +Increasing `time_step` raises the noise level; at large enough values the fit is +dominated by the perturbation and the three basins of the original Branin function +stop being recoverable from this many samples. diff --git a/docs/src/InverseDistance.md b/docs/src/InverseDistance.md index 7e86c66d..13e61ef0 100644 --- a/docs/src/InverseDistance.md +++ b/docs/src/InverseDistance.md @@ -1,4 +1,4 @@ -# InverseDistance Surrogate Tutorial +# Inverse distance surrogate tutorial The **Inverse Distance Surrogate** (Shepard's method) is an interpolating method, and in this method, the unknown points are calculated with a weighted average of the sampling points. This model uses the inverse distance between the unknown and training points to predict the unknown point. We do not need to fit this model because the response of an unknown point x is computed with respect to the distance between x and the training points. @@ -19,6 +19,8 @@ using Surrogates using Plots ``` +## One dimension + ### Sampling We choose to sample f in 100 points between 0 and 10 using the `sample` function. The sampling points are chosen using a Low Discrepancy, this can be done by passing `HaltonSample()` to the `sample` function. @@ -36,7 +38,7 @@ scatter(x, y, label = "Sampled points", xlims = (lower_bound, upper_bound), lege plot!(f, label = "True function", xlims = (lower_bound, upper_bound), legend = :top) ``` -## Building a Surrogate +### Building a Surrogate ```@example Inverse_Distance1D InverseDistance = InverseDistanceSurrogate(x, y, lower_bound, upper_bound) @@ -94,7 +96,7 @@ sampled response exactly. maximum(abs(InverseDistance(x[i]) - y[i]) for i in eachindex(x)) ``` -## Optimizing +### Optimizing Having built a surrogate, we can now use it to search for minima in our original function `f`. @@ -109,7 +111,7 @@ plot!(InverseDistance, label = "Surrogate function", xlims = (lower_bound, upper_bound), legend = :top) ``` -## Inverse Distance Surrogate Tutorial (ND): +## Several dimensions First of all we will define the `Schaffer` function we are going to build a surrogate for. Notice, how its argument is a vector of numbers, one for each coordinate, and its output is a scalar. @@ -141,17 +143,18 @@ zs = schaffer.(xys); ``` ```@example Inverse_DistanceND -x, y = -5:10, 0:15 -p1 = surface(x, y, (x1, x2) -> schaffer((x1, x2))) +xgrid = range(lower_bound[1], upper_bound[1], length = 100) +ygrid = range(lower_bound[2], upper_bound[2], length = 100) +p1 = surface(xgrid, ygrid, (x1, x2) -> schaffer((x1, x2))) xs = [xy[1] for xy in xys] ys = [xy[2] for xy in xys] scatter!(xs, ys, zs) -p2 = contour(x, y, (x1, x2) -> schaffer((x1, x2))) +p2 = contour(xgrid, ygrid, (x1, x2) -> schaffer((x1, x2))) scatter!(xs, ys) plot(p1, p2, title = "True function") ``` -### Building a surrogate +### Building a Surrogate Using the sampled points we build the surrogate, the steps are analogous to the 1-dimensional case. @@ -160,9 +163,9 @@ InverseDistance = InverseDistanceSurrogate(xys, zs, lower_bound, upper_bound) ``` ```@example Inverse_DistanceND -p1 = surface(x, y, (x, y) -> InverseDistance([x y])) +p1 = surface(xgrid, ygrid, (x, y) -> InverseDistance([x y])) scatter!(xs, ys, zs, marker_z = zs) -p2 = contour(x, y, (x, y) -> InverseDistance([x y])) +p2 = contour(xgrid, ygrid, (x, y) -> InverseDistance([x y])) scatter!(xs, ys, marker_z = zs) plot(p1, p2, title = "Surrogate") ``` @@ -189,12 +192,12 @@ length(xys), length(InverseDistance.x) ``` ```@example Inverse_DistanceND -p1 = surface(x, y, (x, y) -> InverseDistance([x y])) +p1 = surface(xgrid, ygrid, (x, y) -> InverseDistance([x y])) xs = [xy[1] for xy in InverseDistance.x] ys = [xy[2] for xy in InverseDistance.x] zs = schaffer.(InverseDistance.x) scatter!(xs, ys, zs, marker_z = zs) -p2 = contour(x, y, (x, y) -> InverseDistance([x y])) +p2 = contour(xgrid, ygrid, (x, y) -> InverseDistance([x y])) scatter!(xs, ys, marker_z = zs) plot(p1, p2) ``` diff --git a/docs/src/LinearSurrogate.md b/docs/src/LinearSurrogate.md index f27ca148..b3f4cfcc 100644 --- a/docs/src/LinearSurrogate.md +++ b/docs/src/LinearSurrogate.md @@ -1,4 +1,4 @@ -## Linear Surrogate +# Linear surrogate tutorial The linear surrogate models a scalar response as an affine function of the explanatory variables: an intercept plus one slope per input dimension, fitted @@ -19,9 +19,11 @@ using Surrogates using Plots ``` +## One dimension + ### Sampling -We choose to sample f in 100 points between 0 and 10 using the `sample` function. The sampling points are chosen using a Sobol sequence, this can be done by passing `SobolSample()` to the `sample` function. +We choose to sample f in 100 points between 5.2 and 12.5 using the `sample` function. The sampling points are chosen using a Sobol sequence, this can be done by passing `SobolSample()` to the `sample` function. ```@example linear_surrogate1D f(x) = 2 * x + 10.0 @@ -34,7 +36,7 @@ scatter(x, y, label = "Sampled points", xlims = (lower_bound, upper_bound)) plot!(f, label = "True function", xlims = (lower_bound, upper_bound)) ``` -## Building a Surrogate +### Building a Surrogate With our sampled points, we can build the **Linear Surrogate** using the `LinearSurrogate` function. @@ -60,7 +62,7 @@ plot!(f, label = "True function", xlims = (lower_bound, upper_bound)) plot!(my_linear_surr_1D, label = "Surrogate function", xlims = (lower_bound, upper_bound)) ``` -## Optimizing +### Optimizing Having built a surrogate, we can now use it to search for minima in our original function `f`. @@ -74,7 +76,7 @@ plot!(f, label = "True function", xlims = (lower_bound, upper_bound)) plot!(my_linear_surr_1D, label = "Surrogate function", xlims = (lower_bound, upper_bound)) ``` -## Vector-valued responses +### Vector-valued responses A single surrogate can model several outputs at once. Pass a vector of responses per sample; the design matrix is shared, so each output is fitted independently @@ -101,7 +103,7 @@ multi.coeff multi(7.0), f(7.0) ``` -## Linear Surrogate tutorial (ND) +## Several dimensions A linear surrogate is the right tool when the response is dominated by a global trend: it is cheap, needs few samples, and its coefficients are directly @@ -137,7 +139,7 @@ y = otl_circuit.(x) extrema(y) ``` -### Building a surrogate +### Building a Surrogate ```@example linear_surrogateND my_linear_ND = LinearSurrogate(x, y, lower_bound, upper_bound) diff --git a/docs/src/Salustowicz.md b/docs/src/Salustowicz.md index 98567122..cbbfff9a 100644 --- a/docs/src/Salustowicz.md +++ b/docs/src/Salustowicz.md @@ -1,7 +1,10 @@ -## Salustowicz Benchmark Function +# Salustowicz Benchmark Function -The true underlying function HyGP had to approximate is the 1D Salustowicz function. The function can be evaluated in the given domain: -``x \in [0, 10]``. +The Salustowicz function is a one-dimensional benchmark, usually evaluated on +``x \in [0, 10]``. The ``e^{-x}x^3`` prefactor makes its amplitude vary by +several orders of magnitude across that domain — large oscillations near +``x \approx 3`` decaying to almost nothing by ``x = 10`` — so it separates +surrogates that adapt their length scale from those that do not. The Salustowicz benchmark function is as follows: @@ -18,9 +21,9 @@ Now, let's define our objective function: ```@example salustowicz1D function salustowicz(x) - term1 = 2.72^(-x) * x^3 * cos(x) * sin(x) + term1 = exp(-x) * x^3 * cos(x) * sin(x) term2 = (cos(x) * sin(x) * sin(x) - 1) - y = term1 * term2 + return term1 * term2 end ``` @@ -30,7 +33,6 @@ Let's sample f in 30 points between 0 and 10 using the `sample` function. The sa n_samples = 30 lower_bound = 0 upper_bound = 10 -num_round = 2 x = sample(n_samples, lower_bound, upper_bound, SobolSample()) y = salustowicz.(x) xs = lower_bound:0.001:upper_bound @@ -51,7 +53,7 @@ plot!(xs, InverseDistance.(xs), label = "InverseDistanceSurrogate", legend = :to plot!(xs, lobachevsky_surrogate.(xs), label = "Lobachevsky", legend = :topright) ``` -Not's let's see Kriging Surrogate with different hyper parameter: +Now let's see the Kriging surrogate under different correlation exponents `p`. Smaller `p` gives rougher sample paths, and the ribbon shows the predicted standard error: ```@example salustowicz1D kriging_surrogate1 = Kriging(x, y, lower_bound, upper_bound, p = 0.9); diff --git a/docs/src/abstractgps.md b/docs/src/abstractgps.md index a173bdbb..45a96b4d 100644 --- a/docs/src/abstractgps.md +++ b/docs/src/abstractgps.md @@ -84,12 +84,13 @@ upper_bound = [1.0, 1.0] xys = sample(n_samples, lower_bound, upper_bound, SobolSample()) zs = hypot_func.(xys) -x, y = -2:2, -2:2 -p1 = surface(x, y, (x1, x2) -> hypot_func((x1, x2))) +xgrid = range(lower_bound[1], upper_bound[1], length = 100) +ygrid = range(lower_bound[2], upper_bound[2], length = 100) +p1 = surface(xgrid, ygrid, (x1, x2) -> hypot_func((x1, x2))) xs = [xy[1] for xy in xys] ys = [xy[2] for xy in xys] scatter!(xs, ys, zs) -p2 = contour(x, y, (x1, x2) -> hypot_func((x1, x2))) +p2 = contour(xgrid, ygrid, (x1, x2) -> hypot_func((x1, x2))) scatter!(xs, ys) plot(p1, p2, title = "True function") ``` @@ -98,9 +99,9 @@ Now let's see how our surrogate performs: ```@example abstractgps_tutorialnd gp_surrogate = AbstractGPSurrogate(xys, zs) -p1 = surface(x, y, (x, y) -> gp_surrogate([x y])) +p1 = surface(xgrid, ygrid, (x, y) -> gp_surrogate([x y])) scatter!(xs, ys, zs, marker_z = zs) -p2 = contour(x, y, (x, y) -> gp_surrogate([x y])) +p2 = contour(xgrid, ygrid, (x, y) -> gp_surrogate([x y])) scatter!(xs, ys, marker_z = zs) plot(p1, p2, title = "Surrogate") ``` diff --git a/docs/src/ackley.md b/docs/src/ackley.md index cb8948d1..98f3b11b 100644 --- a/docs/src/ackley.md +++ b/docs/src/ackley.md @@ -2,7 +2,12 @@ The Ackley function is defined as: ``f(x) = -a*\exp(-b\sqrt{\frac{1}{d}\sum_{i=1}^d x_i^2}) - \exp(\frac{1}{d} \sum_{i=1}^d \cos(cx_i)) + a + \exp(1)`` -Usually the recommended values are: ``a = 20``, ``b = 0.2`` and ``c = 2\pi`` +Usually the recommended values are: ``a = 20``, ``b = 0.2`` and ``c = 2\pi``. + +The global minimum is ``f(0) = 0``. The outer envelope is a shallow, almost flat +bowl, while the cosine term covers it in a fine grid of local minima — so a +surrogate can fit the overall shape easily and still mislead a local optimizer. +That is what makes it a standard test for global methods. Let's see the 1D case. @@ -18,13 +23,13 @@ function ackley(x) a, b, c = 20.0, 0.2, 2.0 * π len_recip = inv(length(x)) sum_sqrs = zero(eltype(x)) - sum_cos = sum_sqrs + sum_cos = zero(eltype(x)) for i in x sum_cos += cos(c * i) sum_sqrs += i^2 end return (-a * exp(-b * sqrt(len_recip * sum_sqrs)) - - exp(len_recip * sum_cos) + a + 2.71) + exp(len_recip * sum_cos) + a + exp(1)) end ``` @@ -47,7 +52,7 @@ my_loba = LobachevskySurrogate(x, y, lb, ub) ```@example ackley scatter(x, y, label = "Sampled points", xlims = (lb, ub), ylims = (0, 30), legend = :top) plot!(xs, ackley.(xs), label = "True function", legend = :top) -plot!(xs, my_rad.(xs), label = "Polynomial expansion", legend = :top) +plot!(xs, my_rad.(xs), label = "Radial basis", legend = :top) plot!(xs, my_loba.(xs), label = "Lobachevsky", legend = :top) ``` diff --git a/docs/src/cantilever.md b/docs/src/cantilever.md index cc6b3834..e889efbd 100644 --- a/docs/src/cantilever.md +++ b/docs/src/cantilever.md @@ -1,8 +1,12 @@ # Cantilever Beam Function The Cantilever Beam function is defined as: -``f(w,t) = \frac{4L^3}{Ewt}*\sqrt{ (\frac{Y}{t^2})^2 + (\frac{X}{w^2})^2 }`` -With parameters L,E,X and Y given. +``f(t,w) = \frac{4L^3}{Ewt}\sqrt{ \left(\frac{Y}{t^2}\right)^2 + \left(\frac{X}{w^2}\right)^2 }`` + +with the input ordered as ``x = (t, w)`` and the beam parameters ``L``, ``E``, +``X`` and ``Y`` fixed below. Note the ``1/(wt)`` factor: the function is singular +along ``t = 0`` and ``w = 0``, so it is evaluated on ``[1, 8]^2``, away from both +axes. Let's import Surrogates and Plots: @@ -32,14 +36,15 @@ Let's plot it: n = 100 lb = [1.0, 1.0] ub = [8.0, 8.0] -xys = sample(n, lb, ub, SobolSample()); -zs = f.(xys); -x, y = 0.0:8.0, 0.0:8.0 -p1 = surface(x, y, (x1, x2) -> f((x1, x2))) +xys = sample(n, lb, ub, SobolSample()) +zs = f.(xys) +xgrid = range(lb[1], ub[1], length = 100) +ygrid = range(lb[2], ub[2], length = 100) +p1 = surface(xgrid, ygrid, (x1, x2) -> f((x1, x2))) xs = [xy[1] for xy in xys] ys = [xy[2] for xy in xys] -scatter!(xs, ys, zs) # hide -p2 = contour(x, y, (x1, x2) -> f((x1, x2))) +scatter!(xs, ys, zs) +p2 = contour(xgrid, ygrid, (x1, x2) -> f((x1, x2))) scatter!(xs, ys) plot(p1, p2, title = "True function") ``` @@ -55,25 +60,25 @@ rad = RadialBasis(xys, zs, lb, ub) Plotting: ```@example beam -p1 = surface(x, y, (x, y) -> mypoly([x y])) +p1 = surface(xgrid, ygrid, (x1, x2) -> mypoly((x1, x2))) scatter!(xs, ys, zs, marker_z = zs) -p2 = contour(x, y, (x, y) -> mypoly([x y])) +p2 = contour(xgrid, ygrid, (x1, x2) -> mypoly((x1, x2))) scatter!(xs, ys, marker_z = zs) plot(p1, p2, title = "Polynomial expansion") ``` ```@example beam -p1 = surface(x, y, (x, y) -> loba([x y])) +p1 = surface(xgrid, ygrid, (x1, x2) -> loba((x1, x2))) scatter!(xs, ys, zs, marker_z = zs) -p2 = contour(x, y, (x, y) -> loba([x y])) +p2 = contour(xgrid, ygrid, (x1, x2) -> loba((x1, x2))) scatter!(xs, ys, marker_z = zs) plot(p1, p2, title = "Lobachevsky") ``` ```@example beam -p1 = surface(x, y, (x, y) -> rad([x y])) +p1 = surface(xgrid, ygrid, (x1, x2) -> rad((x1, x2))) scatter!(xs, ys, zs, marker_z = zs) -p2 = contour(x, y, (x, y) -> rad([x y])) +p2 = contour(xgrid, ygrid, (x1, x2) -> rad((x1, x2))) scatter!(xs, ys, marker_z = zs) -plot(p1, p2, title = "Inverse distance") +plot(p1, p2, title = "Radial basis") ``` diff --git a/docs/src/gek.md b/docs/src/gek.md index 4add4578..11207bca 100644 --- a/docs/src/gek.md +++ b/docs/src/gek.md @@ -27,7 +27,9 @@ using Surrogates using Plots ``` -## Sampling +## One dimension + +### Sampling We choose to sample f in 100 points between 2 and 10 using the `sample` function. The sampling points are chosen using a Sobol sequence, this can be done by passing `SobolSample()` to the `sample` function. @@ -53,7 +55,7 @@ y = vcat(y1, y2) length(y) == 2 * n_samples ``` -## Building a surrogate +### Building a surrogate With our sampled points, we can build the Gradient Enhanced Kriging surrogate using the `GEK` function. @@ -66,7 +68,7 @@ plot!(my_gek, label = "Surrogate function", ribbon = p -> std_error_at_point(my_ xlims = (lower_bound, upper_bound), legend = :top) ``` -# Gradient Enhanced Kriging Surrogate Tutorial (ND) +## Several dimensions First of all, let's define the function we are going to build a surrogate for. @@ -87,7 +89,7 @@ function leon(x) end ``` -## Sampling +### Sampling Let's define our bounds, this time we are working in two dimensions. In particular, we want our first dimension `x` to have bounds `0, 1`, and `0, 1` for the second dimension. We are taking 100 samples of the space using Sobol Sequences. We then evaluate our function on all the sampling points. @@ -100,7 +102,8 @@ y1 = leon.(xys) ``` ```@example GEK_ND -xgrid, ygrid = 0:0.05:1, 0:0.05:1 +xgrid = range(lower_bound[1], upper_bound[1], length = 100) +ygrid = range(lower_bound[2], upper_bound[2], length = 100) p1 = surface(xgrid, ygrid, (x1, x2) -> leon((x1, x2))) xs = [xy[1] for xy in xys] ys = [xy[2] for xy in xys] @@ -110,7 +113,7 @@ scatter!(xs, ys) plot(p1, p2, title = "True function") ``` -## Building a surrogate +### Building a surrogate Using the sampled points, we build the surrogate, the steps are analogous to the 1-dimensional case. diff --git a/docs/src/gramacylee.md b/docs/src/gramacylee.md index 1e3b4d9b..948e8ade 100644 --- a/docs/src/gramacylee.md +++ b/docs/src/gramacylee.md @@ -1,11 +1,18 @@ -## Gramacy & Lee Function +# Gramacy & Lee Function -the Gramacy & Lee function is a continuous function. It is not convex. The function is defined on a 1-dimensional space. It is unimodal. The function can be defined on any input domain, but it is usually evaluated on -``x \in [-0.5, 2.5]``. +The Gramacy & Lee function is a one-dimensional, continuous, non-convex +benchmark. It is **multimodal**: the ``\sin(10\pi x)`` factor puts thirteen local +minima inside the usual evaluation domain +``x \in [-0.5, 2.5]``, which is what makes it a useful test for a surrogate's +ability to resolve fine structure from few samples. -The Gramacy & Lee is as follows: +The Gramacy & Lee function is as follows: ``f(x) = \frac{\sin(10\pi x)}{2x} + (x-1)^4``. +Note that it has a removable singularity at ``x = 0``: the limit is +``5\pi \approx 15.7``, but evaluating the expression there divides zero by zero. +The plotting grid below steps around it. + Let's import these two packages `Surrogates` and `Plots`: ```@example gramacylee1D @@ -20,7 +27,7 @@ Now, let's define our objective function: function gramacylee(x) term1 = sin(10 * pi * x) / (2 * x) term2 = (x - 1)^4 - y = term1 + term2 + return term1 + term2 end ``` @@ -32,7 +39,7 @@ lower_bound = -0.5 upper_bound = 2.5 x = sample(n, lower_bound, upper_bound, SobolSample()) y = gramacylee.(x) -xs = lower_bound:0.001:upper_bound +xs = (lower_bound + 0.0005):0.001:upper_bound scatter(x, y, label = "Sampled points", xlims = (lower_bound, upper_bound), ylims = (-5, 20), legend = :top) plot!(xs, gramacylee.(xs), label = "True function", legend = :top) diff --git a/docs/src/kriging.md b/docs/src/kriging.md index d48efe6a..1c6fc35b 100644 --- a/docs/src/kriging.md +++ b/docs/src/kriging.md @@ -1,4 +1,4 @@ -# Kriging Surrogate Tutorial (1D) +# Kriging surrogate tutorial Kriging, or Gaussian process regression, is a method of interpolation in which the interpolated values are modeled by a Gaussian process. The model has two @@ -26,7 +26,9 @@ using Surrogates using Plots ``` -## Sampling +## One dimension + +### Sampling We choose to sample f in 100 points between 0 and 1 using the `sample` function. The sampling points are chosen using a Sobol sequence; This can be done by passing `SobolSample()` to the `sample` function. @@ -47,7 +49,7 @@ scatter( plot!(xs, f.(xs), label = "True function", legend = :top) ``` -## Building a surrogate +### Building a surrogate With our sampled points, we can build the Kriging surrogate using the `Kriging` function. @@ -63,7 +65,7 @@ plot!(xs, kriging_surrogate.(xs), label = "Surrogate function", ribbon = p -> std_error_at_point(kriging_surrogate, p), legend = :top) ``` -## Optimizing +### Optimizing Having built a surrogate, we can now use it to search for minima in our original function `f`. @@ -79,7 +81,7 @@ plot!(xs, kriging_surrogate.(xs), label = "Surrogate function", ribbon = p -> std_error_at_point(kriging_surrogate, p), legend = :top) ``` -# Kriging Surrogate Tutorial (ND) +## Several dimensions First of all, let's define the function we are going to build a surrogate for. Notice how its argument is a vector of numbers, one for each coordinate, and its output is a scalar. @@ -101,7 +103,7 @@ function branin(x) end ``` -## Sampling +### Sampling Let's define our bounds, this time we are working in two dimensions. In particular, we want our first dimension `x` to have bounds `-5, 10`, and `0, 15` for the second dimension. We are taking 50 samples of the space using Sobol sequences. We then evaluate our function on all the sampling points. @@ -115,17 +117,18 @@ zs = branin.(xys) ``` ```@example kriging_tutorialnd -x, y = -5:10, 0:15 -p1 = surface(x, y, (x1, x2) -> branin((x1, x2))) +xgrid = range(lower_bound[1], upper_bound[1], length = 100) +ygrid = range(lower_bound[2], upper_bound[2], length = 100) +p1 = surface(xgrid, ygrid, (x1, x2) -> branin((x1, x2))) xs = [xy[1] for xy in xys] ys = [xy[2] for xy in xys] scatter!(xs, ys, zs) -p2 = contour(x, y, (x1, x2) -> branin((x1, x2))) +p2 = contour(xgrid, ygrid, (x1, x2) -> branin((x1, x2))) scatter!(xs, ys) plot(p1, p2, title = "True function") ``` -## Building a surrogate +### Building a surrogate Using the sampled points, we build the surrogate, the steps are analogous to the 1-dimensional case. @@ -139,14 +142,14 @@ kriging_surrogate.theta ``` ```@example kriging_tutorialnd -p1 = surface(x, y, (x, y) -> kriging_surrogate([x y])) +p1 = surface(xgrid, ygrid, (x, y) -> kriging_surrogate([x y])) scatter!(xs, ys, zs, marker_z = zs) -p2 = contour(x, y, (x, y) -> kriging_surrogate([x y])) +p2 = contour(xgrid, ygrid, (x, y) -> kriging_surrogate([x y])) scatter!(xs, ys, marker_z = zs) plot(p1, p2, title = "Surrogate") ``` -## Optimizing +### Optimizing With our surrogate, we can now search for the minima of the branin function. @@ -167,12 +170,12 @@ size(xys) ``` ```@example kriging_tutorialnd -p1 = surface(x, y, (x, y) -> kriging_surrogate([x y])) +p1 = surface(xgrid, ygrid, (x, y) -> kriging_surrogate([x y])) xs = [xy[1] for xy in xys] ys = [xy[2] for xy in xys] zs = branin.(xys) scatter!(xs, ys, zs, marker_z = zs) -p2 = contour(x, y, (x, y) -> kriging_surrogate([x y])) +p2 = contour(xgrid, ygrid, (x, y) -> kriging_surrogate([x y])) scatter!(xs, ys, marker_z = zs) plot(p1, p2) ``` diff --git a/docs/src/lobachevsky.md b/docs/src/lobachevsky.md index 05bb97e4..9d01bc82 100644 --- a/docs/src/lobachevsky.md +++ b/docs/src/lobachevsky.md @@ -25,9 +25,11 @@ using Surrogates using Plots ``` -## Sampling +## One dimension -We choose to sample f in 100 points between 0 and 4 using the `sample` function. The sampling points are chosen using a Sobol sequence, this can be done by passing `SobolSample()` to the `sample` function. +### Sampling + +We choose to sample f in 100 points between 1 and 4 using the `sample` function. The sampling points are chosen using a Sobol sequence, this can be done by passing `SobolSample()` to the `sample` function. ```@example LobachevskySurrogate_tutorial f(x) = sin(x) + sin(10 / 3 * x) @@ -40,7 +42,7 @@ scatter(x, y, label = "Sampled points", xlims = (lower_bound, upper_bound)) plot!(f, label = "True function", xlims = (lower_bound, upper_bound)) ``` -## Building a surrogate +### Building a surrogate With our sampled points, we can build the Lobachevsky surrogate using the `LobachevskySurrogate` function. @@ -58,7 +60,7 @@ plot!( lobachevsky_surrogate, label = "Surrogate function", xlims = (lower_bound, upper_bound)) ``` -## Optimizing +### Optimizing Having built a surrogate, we can now use it to search for minima in our original function `f`. @@ -73,7 +75,7 @@ plot!( lobachevsky_surrogate, label = "Surrogate function", xlims = (lower_bound, upper_bound)) ``` -## The closed-form integral +### The closed-form integral The Lobachevsky spline integrates in closed form, which the other surrogates in this package do not. `lobachevsky_integral` evaluates that formula, and it @@ -94,7 +96,7 @@ println("relative difference = ", abs(closed - numerical) / abs(numerical)) In the example below, it shows how to use `lobachevsky_surrogate` for higher dimension problems. -# Lobachevsky Surrogate Tutorial (ND): +## Several dimensions First of all, we will define the `Schaffer` function we are going to build surrogate for. Notice, one how its argument is a vector of numbers, one for each coordinate, and its output is a scalar. @@ -112,7 +114,7 @@ function schaffer(x) end ``` -## Sampling +### Sampling Let's define our bounds, this time we are working in two dimensions. In particular, we want our first dimension `x` to have bounds `0, 8`, and `0, 8` for the second dimension. We are taking 60 samples of the space using Sobol Sequences. We then evaluate our function on all of the sampling points. @@ -126,17 +128,18 @@ zs = schaffer.(xys); ``` ```@example LobachevskySurrogate_ND -x, y = 0:8, 0:8 -p1 = surface(x, y, (x1, x2) -> schaffer((x1, x2))) +xgrid = range(lower_bound[1], upper_bound[1], length = 100) +ygrid = range(lower_bound[2], upper_bound[2], length = 100) +p1 = surface(xgrid, ygrid, (x1, x2) -> schaffer((x1, x2))) xs = [xy[1] for xy in xys] ys = [xy[2] for xy in xys] scatter!(xs, ys, zs) -p2 = contour(x, y, (x1, x2) -> schaffer((x1, x2))) +p2 = contour(xgrid, ygrid, (x1, x2) -> schaffer((x1, x2))) scatter!(xs, ys) plot(p1, p2, title = "True function") ``` -## Building a surrogate +### Building a surrogate Using the sampled points, we build the surrogate, the steps are analogous to the 1-dimensional case. @@ -146,14 +149,14 @@ Lobachevsky = LobachevskySurrogate( ``` ```@example LobachevskySurrogate_ND -p1 = surface(x, y, (x, y) -> Lobachevsky([x y])) +p1 = surface(xgrid, ygrid, (x, y) -> Lobachevsky([x y])) scatter!(xs, ys, zs, marker_z = zs) -p2 = contour(x, y, (x, y) -> Lobachevsky([x y])) +p2 = contour(xgrid, ygrid, (x, y) -> Lobachevsky([x y])) scatter!(xs, ys, marker_z = zs) plot(p1, p2, title = "Surrogate") ``` -## Optimizing +### Optimizing With our surrogate, we can now search for the minima of the function. @@ -175,18 +178,18 @@ length(xys), length(Lobachevsky.x) ``` ```@example LobachevskySurrogate_ND -p1 = surface(x, y, (x, y) -> Lobachevsky([x y])) +p1 = surface(xgrid, ygrid, (x, y) -> Lobachevsky([x y])) xys = Lobachevsky.x xs = [i[1] for i in xys] ys = [i[2] for i in xys] zs = schaffer.(xys) scatter!(xs, ys, zs, marker_z = zs) -p2 = contour(x, y, (x, y) -> Lobachevsky([x y])) +p2 = contour(xgrid, ygrid, (x, y) -> Lobachevsky([x y])) scatter!(xs, ys, marker_z = zs) plot(p1, p2) ``` -## Integrating out a coordinate +### Integrating out a coordinate `lobachevsky_integrate_dimension` integrates the surrogate over one coordinate and returns a surrogate on the remaining ones — the marginal. Here we integrate @@ -213,7 +216,7 @@ lobachevsky_integral(marginal, marginal.lb, marginal.ub), lobachevsky_integral(Lobachevsky, lower_bound, upper_bound) ``` -## Vector-valued responses +### Vector-valued responses The interpolant is linear in the responses and the kernel matrix does not involve them, so several outputs can be fitted at once against the same matrix. diff --git a/docs/src/lp.md b/docs/src/lp.md index ae3613a6..313dc759 100644 --- a/docs/src/lp.md +++ b/docs/src/lp.md @@ -20,7 +20,9 @@ function f(x, p) end ``` -Let's see a simple 1D case: +In one dimension the Lp norm collapses to ``|x|`` for every ``p``, so the +exponent only starts to matter from two dimensions on. The 1D case below is +therefore just the absolute value: ```@example lp n = 30 @@ -28,7 +30,7 @@ lb = -5.0 ub = 5.0 p = 1.3 x = sample(n, lb, ub, SobolSample()) -y = f.(x, p) +y = f.(x, p) # = abs.(x), independent of p in one dimension xs = lb:0.001:ub plot(x, y, seriestype = :scatter, label = "Sampled points", xlims = (lb, ub), ylims = (0, 5), legend = :top) diff --git a/docs/src/moe.md b/docs/src/moe.md index 21d839a1..f24a376a 100644 --- a/docs/src/moe.md +++ b/docs/src/moe.md @@ -58,7 +58,7 @@ MOE_at0 = MOE_1D_RAD_RAD(0.0) As we can see, the accuracy is significantly better. -## Under the Hood - How SurrogatesMOE Works +## Under the Hood: How SurrogatesMOE Works First, we create Gaussian Mixture Models for the number of expert types provided using the x and y values. For example, in the above example, we create two clusters. Then, using a small test dataset kept aside from the input data, we choose the best surrogate model for each of the clusters. At prediction time, we use the appropriate surrogate model based on the cluster to which the new point belongs. @@ -111,7 +111,7 @@ rbf_rmse = rmse(true_vals, rbf_pred_vals) @show rbf_rmse, moe_rmse ``` -## Usage Notes - Example With Other Surrogates +## Usage Notes: Example With Other Surrogates From the above example, simply change or add to the expert types: diff --git a/docs/src/multi_objective_opt.md b/docs/src/multi_objective_opt.md index 8351e536..50f01166 100644 --- a/docs/src/multi_objective_opt.md +++ b/docs/src/multi_objective_opt.md @@ -44,7 +44,7 @@ ub = [3.5, 0.5] x = sample(50, lb, ub, SobolSample()) y = f.(x) my_radial_basis_ego = RadialBasis(x, y, lb, ub) -#I can find my pareto set and pareto front by calling again the surrogate_optimize! function: +# The Pareto set and Pareto front come back from surrogate_optimize!: pareto_set, pareto_front = surrogate_optimize!( f, SMB(), lb, ub, my_radial_basis_ego, SobolSample(); maxiters = 10, n_new_look = 100); diff --git a/docs/src/neural.md b/docs/src/neural.md index cc5830ce..e3075d81 100644 --- a/docs/src/neural.md +++ b/docs/src/neural.md @@ -40,12 +40,13 @@ zs = schaffer.(xys) ``` ```@example Neural_surrogate -x, y = 0:8, 0:8 -p1 = surface(x, y, (x1, x2) -> schaffer((x1, x2))) +xgrid = range(lower_bound[1], upper_bound[1], length = 100) +ygrid = range(lower_bound[2], upper_bound[2], length = 100) +p1 = surface(xgrid, ygrid, (x1, x2) -> schaffer((x1, x2))) xs = [xy[1] for xy in xys] ys = [xy[2] for xy in xys] scatter!(xs, ys, zs) -p2 = contour(x, y, (x1, x2) -> schaffer((x1, x2))) +p2 = contour(xgrid, ygrid, (x1, x2) -> schaffer((x1, x2))) scatter!(xs, ys) plot(p1, p2, title = "True function") ``` @@ -66,9 +67,9 @@ neural = NeuralSurrogate(xys, zs, lower_bound, upper_bound, model = model1, n_ep ``` ```@example Neural_surrogate -p1 = surface(x, y, (x, y) -> neural([x, y])) +p1 = surface(xgrid, ygrid, (x, y) -> neural([x, y])) scatter!(xs, ys, zs, marker_z = zs) -p2 = contour(x, y, (x, y) -> neural([x, y])) +p2 = contour(xgrid, ygrid, (x, y) -> neural([x, y])) scatter!(xs, ys, marker_z = zs) plot(p1, p2, title = "Surrogate") ``` diff --git a/docs/src/polychaos.md b/docs/src/polychaos.md index a6f74174..752b08b0 100644 --- a/docs/src/polychaos.md +++ b/docs/src/polychaos.md @@ -8,7 +8,7 @@ PolynomialChaosSurrogate ## Sampling -We choose to sample f in 100 points between 0 and 10 using the `sample` function. The sampling points are chosen using a Low Discrepancy. This can be done by passing `HaltonSample()` to the `sample` function. +We choose to sample f in 100 points between 1 and 6 using the `sample` function. The sampling points are chosen from a low-discrepancy sequence, by passing `HaltonSample()` to the `sample` function. ```@example polychaos using Surrogates diff --git a/docs/src/radials.md b/docs/src/radials.md index 7f208682..5ae701ed 100644 --- a/docs/src/radials.md +++ b/docs/src/radials.md @@ -1,4 +1,4 @@ -# Radial Basis Surrogates Tutorial (1D) +# Radial basis surrogate tutorial The Radial Basis Surrogate model represents the interpolating function as a linear combination of basis functions, one for each training point. Let's say we are building a surrogate for: @@ -33,7 +33,9 @@ scatter(x, y, label = "Sampled Points", xlims = (lower_bound, upper_bound), lege plot!(x, y, label = "True function", legend = :top) ``` -## Building Surrogate +## One dimension + +### Building a surrogate With our sampled points we can build the **Radial Surrogate** using the `RadialBasis` function. @@ -63,7 +65,7 @@ plot!(radial_surrogate, label = "Surrogate function", xlims = (lower_bound, upper_bound), legend = :top) ``` -## Optimizing +### Optimizing Having built a surrogate, we can now use it to search for minima in our original function `f`. @@ -78,7 +80,7 @@ plot!(radial_surrogate, label = "Surrogate function", xlims = (lower_bound, upper_bound), legend = :top) ``` -# Radial Basis Surrogate Tutorial (ND) +## Several dimensions First of all, we will define the `Booth` function we are going to build the surrogate for: @@ -100,7 +102,7 @@ function booth(x) end ``` -## Sampling +### Sampling Let's define our bounds, this time we are working in two dimensions. In particular we want our first dimension `x` to have bounds `-5, 10`, and `0, 15` for the second dimension. We are taking 100 samples of the space using Sobol Sequences. We then evaluate our function on all of the sampling points. @@ -114,17 +116,18 @@ zs = booth.(xys) ``` ```@example RadialBasisSurrogateND -x, y = -5.0:10.0, 0.0:15.0 -p1 = surface(x, y, (x1, x2) -> booth((x1, x2))) +xgrid = range(lower_bound[1], upper_bound[1], length = 100) +ygrid = range(lower_bound[2], upper_bound[2], length = 100) +p1 = surface(xgrid, ygrid, (x1, x2) -> booth((x1, x2))) xs = [xy[1] for xy in xys] ys = [xy[2] for xy in xys] scatter!(xs, ys, zs) -p2 = contour(x, y, (x1, x2) -> booth((x1, x2))) +p2 = contour(xgrid, ygrid, (x1, x2) -> booth((x1, x2))) scatter!(xs, ys) plot(p1, p2, title = "True function") ``` -## Building a surrogate +### Building a surrogate Using the sampled points we build the surrogate, the steps are analogous to the 1-dimensional case. @@ -133,14 +136,14 @@ radial_basis = RadialBasis(xys, zs, lower_bound, upper_bound) ``` ```@example RadialBasisSurrogateND -p1 = surface(x, y, (x, y) -> radial_basis([x y])) +p1 = surface(xgrid, ygrid, (x, y) -> radial_basis([x y])) scatter!(xs, ys, zs, marker_z = zs) -p2 = contour(x, y, (x, y) -> radial_basis([x y])) +p2 = contour(xgrid, ygrid, (x, y) -> radial_basis([x y])) scatter!(xs, ys, marker_z = zs) plot(p1, p2, title = "Surrogate") ``` -## Optimizing +### Optimizing With our surrogate, we can now search for the minima of the function. @@ -161,12 +164,12 @@ size(xys) ``` ```@example RadialBasisSurrogateND -p1 = surface(x, y, (x, y) -> radial_basis([x y])) +p1 = surface(xgrid, ygrid, (x, y) -> radial_basis([x y])) xs = [xy[1] for xy in xys] ys = [xy[2] for xy in xys] zs = booth.(xys) scatter!(xs, ys, zs, marker_z = zs) -p2 = contour(x, y, (x, y) -> radial_basis([x y])) +p2 = contour(xgrid, ygrid, (x, y) -> radial_basis([x y])) scatter!(xs, ys, marker_z = zs) plot(p1, p2) ``` diff --git a/docs/src/rosenbrock.md b/docs/src/rosenbrock.md index 96d0a28a..95911295 100644 --- a/docs/src/rosenbrock.md +++ b/docs/src/rosenbrock.md @@ -1,9 +1,12 @@ # Rosenbrock function The Rosenbrock function is defined as: -``f(x) = \sum_{i=1}^{d-1}[ (x_{i+1}-x_i)^2 + (x_i - 1)^2]`` +``f(x) = \sum_{i=1}^{d-1}[ 100(x_{i+1}-x_i^2)^2 + (x_i - 1)^2]`` -I will treat the 2D version, which is commonly defined as: +Its global minimum, ``f(\mathbf{1}) = 0``, sits at the bottom of a narrow curved +valley: easy to enter and hard to traverse, which is what the benchmark tests. + +We treat the 2D version here, which is commonly written as: ``f(x,y) = (1-x)^2 + 100(y-x^2)^2`` Let's import Surrogates and Plots: @@ -31,12 +34,13 @@ lb = [0.0, 0.0] ub = [1.0, 1.0] xys = sample(n, lb, ub, SobolSample()) zs = f.(xys); -x, y = 0:1, 0:1 -p1 = surface(x, y, (x1, x2) -> f((x1, x2))) +xgrid = range(lb[1], ub[1], length = 100) +ygrid = range(lb[2], ub[2], length = 100) +p1 = surface(xgrid, ygrid, (x1, x2) -> f((x1, x2))) xs = [xy[1] for xy in xys] ys = [xy[2] for xy in xys] scatter!(xs, ys, zs) -p2 = contour(x, y, (x1, x2) -> f((x1, x2))) +p2 = contour(xgrid, ygrid, (x1, x2) -> f((x1, x2))) scatter!(xs, ys) plot(p1, p2, title = "True function") ``` @@ -52,25 +56,25 @@ inver = InverseDistanceSurrogate(xys, zs, lb, ub) Plotting: ```@example rosen -p1 = surface(x, y, (x, y) -> mypoly([x y])) +p1 = surface(xgrid, ygrid, (x, y) -> mypoly([x y])) scatter!(xs, ys, zs, marker_z = zs) -p2 = contour(x, y, (x, y) -> mypoly([x y])) +p2 = contour(xgrid, ygrid, (x, y) -> mypoly([x y])) scatter!(xs, ys, marker_z = zs) plot(p1, p2, title = "Polynomial expansion") ``` ```@example rosen -p1 = surface(x, y, (x, y) -> loba([x y])) +p1 = surface(xgrid, ygrid, (x, y) -> loba([x y])) scatter!(xs, ys, zs, marker_z = zs) -p2 = contour(x, y, (x, y) -> loba([x y])) +p2 = contour(xgrid, ygrid, (x, y) -> loba([x y])) scatter!(xs, ys, marker_z = zs) plot(p1, p2, title = "Lobachevsky") ``` ```@example rosen -p1 = surface(x, y, (x, y) -> inver([x y])) +p1 = surface(xgrid, ygrid, (x, y) -> inver([x y])) scatter!(xs, ys, zs, marker_z = zs) -p2 = contour(x, y, (x, y) -> inver([x y])) +p2 = contour(xgrid, ygrid, (x, y) -> inver([x y])) scatter!(xs, ys, marker_z = zs) plot(p1, p2, title = "Inverse distance") ``` diff --git a/docs/src/sphere_function.md b/docs/src/sphere_function.md index 177f6c6b..57d15545 100644 --- a/docs/src/sphere_function.md +++ b/docs/src/sphere_function.md @@ -2,7 +2,10 @@ The sphere function of dimension d is defined as: ``f(x) = \sum_{i=1}^d x_i^2`` -with lower bound -10 and upper bound 10. +with lower bound -10 and upper bound 10. Its global minimum is +``f(\mathbf{0}) = 0``. It is convex and perfectly smooth, which makes it the +sanity check of the benchmark set: a surrogate that struggles here has a +problem unrelated to the difficulty of the objective. Let's import Surrogates and Plots: @@ -33,7 +36,7 @@ plot(x, y, seriestype = :scatter, label = "Sampled points", plot!(xs, sphere_function.(xs), label = "True function", legend = :top) ``` -Fitting RadialSurrogate with different radial basis: +Fitting `RadialBasis` with different radial basis functions: ```@example sphere_function rad_1d_linear = RadialBasis(x, y, lb, ub) diff --git a/docs/src/surrogate.md b/docs/src/surrogate.md index d6fed66c..1f6d1103 100644 --- a/docs/src/surrogate.md +++ b/docs/src/surrogate.md @@ -122,7 +122,7 @@ WendlandStructure PolyChaosStructure ``` -# Creating another surrogate +## Creating another surrogate It's great that you want to add another surrogate to the library! You will need to: @@ -131,7 +131,7 @@ You will need to: 2. Define update!(your\_surrogate, x\_new, y\_new) 3. Define your\_surrogate(value) for the approximation -## Example +### Example ```julia mutable struct NewSurrogate{X, Y, L, U, C, A, B} <: AbstractDeterministicSurrogate diff --git a/docs/src/tensor_prod.md b/docs/src/tensor_prod.md index 98dff225..eacdcac3 100644 --- a/docs/src/tensor_prod.md +++ b/docs/src/tensor_prod.md @@ -23,7 +23,6 @@ end n = 100 lb = -5.0 ub = 5.0 -a = 0.5 x = sample(n, lb, ub, SobolSample()) y = f.(x) xs = lb:0.001:ub diff --git a/docs/src/water_flow.md b/docs/src/water_flow.md index 97227fa9..7d0baeba 100644 --- a/docs/src/water_flow.md +++ b/docs/src/water_flow.md @@ -3,7 +3,9 @@ The water flow function is defined as: ``f(r_w,r,T_u,H_u,T_l,H_l,L,K_w) = \frac{2*\pi*T_u(H_u - H_l)}{\log(\frac{r}{r_w})*[1 + \frac{2LT_u}{\log(\frac{r}{r_w})*r_w^2*K_w}+ \frac{T_u}{T_l} ]}`` -It has 8 dimensions. +It has 8 dimensions, with the bounds below. The response spans roughly an +order of magnitude across that box, so the errors reported at the end are worth +reading against that scale rather than in absolute terms. ```@example water using Surrogates @@ -32,7 +34,6 @@ end ```@example water n = 180 -d = 8 lb = [0.05, 100, 63070, 990, 63.1, 700, 1120, 9855] ub = [0.15, 50000, 115600, 1110, 116, 820, 1680, 12045] x = sample(n, lb, ub, SobolSample()) @@ -47,8 +48,8 @@ my_rad = RadialBasis(x, y, lb, ub) y_rad = my_rad.(x_test) my_poly = PolynomialChaosSurrogate(x, y, lb, ub) y_poly = my_poly.(x_test) -mse_rad = norm(y_true - y_rad, 2) / n_test -mse_poly = norm(y_true - y_poly, 2) / n_test -println("MSE Radial: $mse_rad") -println("MSE Radial: $mse_poly") +rmse_rad = norm(y_true - y_rad, 2) / sqrt(n_test) +rmse_poly = norm(y_true - y_poly, 2) / sqrt(n_test) +println("RMSE Radial: $rmse_rad") +println("RMSE Polynomial: $rmse_poly") ``` diff --git a/docs/src/welded_beam.md b/docs/src/welded_beam.md index df9bd0ce..7125cc8f 100644 --- a/docs/src/welded_beam.md +++ b/docs/src/welded_beam.md @@ -6,7 +6,7 @@ With: ``a = \frac{6000}{\sqrt{2}hl}`` ``b = \frac{6000(14 + 0.5l)*\sqrt{0.25(l^2+(h+t)^2)}}{2*[0.707hl(\frac{l^2}{12}+0.25*(h+t)^2)]}`` -It has 3 dimension. +It has 3 dimensions. ```@example welded using Surrogates @@ -30,7 +30,6 @@ end ```@example welded n = 300 -d = 3 lb = [0.125, 5.0, 5.0] ub = [1.0, 10.0, 10.0] x = sample(n, lb, ub, SobolSample()) @@ -43,11 +42,11 @@ y_true = f.(x_test) ```@example welded my_rad = RadialBasis(x, y, lb, ub) y_rad = my_rad.(x_test) -mse_rad = norm(y_true - y_rad, 2) / n_test -println("MSE Radial: $mse_rad") +rmse_rad = norm(y_true - y_rad, 2) / sqrt(n_test) +println("RMSE Radial: $rmse_rad") my_loba = LobachevskySurrogate(x, y, lb, ub) y_loba = my_loba.(x_test) -mse_rad = norm(y_true - y_loba, 2) / n_test -println("MSE Lobachevsky: $mse_rad") +rmse_loba = norm(y_true - y_loba, 2) / sqrt(n_test) +println("RMSE Lobachevsky: $rmse_loba") ``` diff --git a/docs/src/wendland.md b/docs/src/wendland.md index 4c251a93..f8baecea 100644 --- a/docs/src/wendland.md +++ b/docs/src/wendland.md @@ -32,7 +32,9 @@ x = sample(n, lower_bound, upper_bound, SobolSample()) y = f.(x) ``` -## Building Surrogate +## One dimension + +### Building a surrogate `eps` is the reciprocal of the support radius: a sample point influences predictions within distance `1 / eps` of it, and nowhere else. Here `eps = 0.45` @@ -80,7 +82,7 @@ end plot!() ``` -## Wendland Surrogate Tutorial (ND) +## Several dimensions The surrogate works the same way in more dimensions; only the bounds change shape. Note that the kernel exponent depends on the input dimension, so the @@ -115,10 +117,11 @@ maximum(abs(wend_ND(xys[i]) - zs[i]) for i in eachindex(xys)) ```@example wendlandND xs = [xy[1] for xy in xys] ys = [xy[2] for xy in xys] -x, y = -5.0:10.0, 0.0:15.0 -p1 = surface(x, y, (x, y) -> wend_ND([x y])) +xgrid = range(lower_bound[1], upper_bound[1], length = 100) +ygrid = range(lower_bound[2], upper_bound[2], length = 100) +p1 = surface(xgrid, ygrid, (x, y) -> wend_ND([x y])) scatter!(xs, ys, zs, marker_z = zs) -p2 = contour(x, y, (x, y) -> wend_ND([x y])) +p2 = contour(xgrid, ygrid, (x, y) -> wend_ND([x y])) scatter!(xs, ys, marker_z = zs) plot(p1, p2, title = "Wendland surrogate") ``` diff --git a/docs/src/xgboost.md b/docs/src/xgboost.md index 9d0dc0ef..84497a39 100644 --- a/docs/src/xgboost.md +++ b/docs/src/xgboost.md @@ -1,4 +1,4 @@ -## XGBoost Surrogate Tutorial +# XGBoost Surrogate Tutorial Gradient boosting fits an additive ensemble of decision trees, each round training a new tree on the residuals of the ensemble so far. @@ -16,9 +16,11 @@ using XGBoost using Plots ``` +## One dimension + ### Sampling -We choose to sample f in 100 points between 0 and 1 using the `sample` function. The sampling points are chosen using a Sobol sequence, this can be done by passing `SobolSample()` to the `sample` function. +We choose to sample f in 100 points between 2.7 and 7.5 using the `sample` function. The sampling points are chosen using a Sobol sequence, this can be done by passing `SobolSample()` to the `sample` function. ```@example XGBoostSurrogate_tutorial f(x) = sin(x) + sin(10 / 3 * x) @@ -63,7 +65,7 @@ plot!(xgboost_surrogate, label = "Surrogate function", xlims = (lower_bound, upper_bound), legend = :top) ``` -## XGBoost ND +## Several dimensions First of all we will define the `Bukin Function N. 6` function we are going to build a surrogate for. @@ -95,12 +97,13 @@ zs = bukin6.(xys) ``` ```@example XGBoostSurrogateND -x, y = -5:10, 0:15 -p1 = surface(x, y, (x1, x2) -> bukin6((x1, x2))) +xgrid = range(lower_bound[1], upper_bound[1], length = 100) +ygrid = range(lower_bound[2], upper_bound[2], length = 100) +p1 = surface(xgrid, ygrid, (x1, x2) -> bukin6((x1, x2))) xs = [xy[1] for xy in xys] ys = [xy[2] for xy in xys] scatter!(xs, ys, zs) -p2 = contour(x, y, (x1, x2) -> bukin6((x1, x2))) +p2 = contour(xgrid, ygrid, (x1, x2) -> bukin6((x1, x2))) scatter!(xs, ys) plot(p1, p2, title = "True function") ``` @@ -114,9 +117,9 @@ XGBoost = XGBoostSurrogate(xys, zs, lower_bound, upper_bound) ``` ```@example XGBoostSurrogateND -p1 = surface(x, y, (x, y) -> XGBoost([x y])) +p1 = surface(xgrid, ygrid, (x, y) -> XGBoost([x y])) scatter!(xs, ys, zs, marker_z = zs) -p2 = contour(x, y, (x, y) -> XGBoost([x y])) +p2 = contour(xgrid, ygrid, (x, y) -> XGBoost([x y])) scatter!(xs, ys, marker_z = zs) plot(p1, p2, title = "Surrogate") ``` @@ -142,12 +145,12 @@ size(xys) ``` ```@example XGBoostSurrogateND -p1 = surface(x, y, (x, y) -> XGBoost([x y])) +p1 = surface(xgrid, ygrid, (x, y) -> XGBoost([x y])) xs = [xy[1] for xy in xys] ys = [xy[2] for xy in xys] zs = bukin6.(xys) scatter!(xs, ys, zs, marker_z = zs) -p2 = contour(x, y, (x, y) -> XGBoost([x y])) +p2 = contour(xgrid, ygrid, (x, y) -> XGBoost([x y])) scatter!(xs, ys, marker_z = zs) plot(p1, p2) ```