From 2dc6141134b77af618e1c96bdd5158146bdcd134 Mon Sep 17 00:00:00 2001 From: simon_coter Date: Mon, 17 Aug 2026 17:15:13 +0200 Subject: [PATCH] removing obsolete OCNE 1.x release --- OCNE/.env | 94 ---- OCNE/.gitattributes | 1 - OCNE/.gitignore | 10 - OCNE/README.md | 204 --------- OCNE/Vagrantfile | 349 -------------- OCNE/scripts/provision.sh | 942 -------------------------------------- 6 files changed, 1600 deletions(-) delete mode 100644 OCNE/.env delete mode 100644 OCNE/.gitattributes delete mode 100644 OCNE/.gitignore delete mode 100644 OCNE/README.md delete mode 100644 OCNE/Vagrantfile delete mode 100755 OCNE/scripts/provision.sh diff --git a/OCNE/.env b/OCNE/.env deleted file mode 100644 index 6349ed1b..00000000 --- a/OCNE/.env +++ /dev/null @@ -1,94 +0,0 @@ -# -*- mode: shell-script -*- -# vi: set ft=shell : - -# Oracle Cloud Native Environment configuration file -# -# Requires vagrant-env plugin -# -# This file will be overwritten on updates, it is recommended to make changes -# in .env.local - -# Verbose console -# VERBOSE=false - -# Set the VM provider host-only / private network subnet -# SUBNET=192.168.99 - -# Set vCPU count and memory for the VMs: -# + 2 vCPU/1770MB absloute memory minimum for Master node(s) -# + 1 vCPU/700MB absloute memory minimum for Worker node(s) -# + 3GB memory minimum required for Istio module on Worker nodes -# OPERATOR_CPUS=1 -# OPERATOR_MEMORY=1024 -# CONTROL_PLANE_CPUS=2 -# CONTROL_PLANE_MEMORY=2048 -# WORKER_CPUS=1 -# WORKER_MEMORY=1024 - -# Group VirtualBox containers -# VB_GROUP="OCNE" - -# Create a separate instance for the operator node? -# The default is to install the Platform API Server and CLI tool on -# the (first) control_plane node -# STANDALONE_OPERATOR=false - -# Multi-control_plane setup. Deploy 3 control_planes in HA mode. -# Will automatically enable STANDALONE_OPERATOR if true -# MULTI_CONTROL_PLANE=false - -# Creates an extra disk (/dev/sdb) so it can be used as a -# Gluster Storage for Kubernetes Persistent Volumes -# EXTRA_DISK=false - -# Number of worker nodes to provision -# NB_WORKERS=2 - -# Bind the kubectl proxy from the (first) control_plane to the vagrant host -# BIND_PROXY=false - -# Additional yum channel to consider (e.g. local repo) -# YUM_REPO= - -# Add OCNE developer channel -# OCNE_DEV=false - -# Container registry for Oracle Cloud Native Environment images -# You can use registry mirrors in a region close to you. -# Check the README.md file for more details. -# REGISTRY_OCNE='container-registry.oracle.com/olcne' - -# Environment and cluster names -# OCNE_ENV_NAME="ocne-env" -# OCNE_CLUSTER_NAME="ocne-cluster" - -# Deploy Calico networking module? Place your config file in the current directory named: calico-config.yaml -# DEPLOY_CALICO=false -# CALICO_MODULE_NAME="ocne-calico" - -# Deploy Multus networking module? Place your config file in the current directory named: multus-config.conf -# DEPLOY_MULTUS=false -# MULTUS_MODULE_NAME="ocne-multus" - -# Deploy the Helm module? (deprecated) -# DEPLOY_HELM=false -# HELM_MODULE_NAME="ocne-helm" - -# Deploy the Istio module? -# DEPLOY_ISTIO=false -# ISTIO_MODULE_NAME="ocne-istio" - -# Deploy the Gluster module? (deprecated) -# DEPLOY_GLUSTER=false -# GLUSTER_MODULE_NAME="ocne-gluster" - -# Deploy the MetalLB module? -# DEPLOY_METALLB=false -# METALLB_MODULE_NAME="ocne-metallb" - -# Override number of control_planes to deploy -# This should not be changed -- for development purpose -# NB_CONTROL_PLANES=1 - -# Update Base OS (experimental) -# UPDATE_OS=false diff --git a/OCNE/.gitattributes b/OCNE/.gitattributes deleted file mode 100644 index 105b5702..00000000 --- a/OCNE/.gitattributes +++ /dev/null @@ -1 +0,0 @@ -*.sh text eol=lf diff --git a/OCNE/.gitignore b/OCNE/.gitignore deleted file mode 100644 index a0283743..00000000 --- a/OCNE/.gitignore +++ /dev/null @@ -1,10 +0,0 @@ -.env.local* -admin.conf -local -id_rsa* -known_hosts* -*.swp -heketi.json* -topology-olcne.json* -hyperconverged.yaml* -ignore-systemd-session-slice.conf* diff --git a/OCNE/README.md b/OCNE/README.md deleted file mode 100644 index b3419d8d..00000000 --- a/OCNE/README.md +++ /dev/null @@ -1,204 +0,0 @@ -# Vagrant project to set up Oracle Cloud Native Environment on Oracle Linux 8 - -Due to major changes in OCNE 2.0 release this Vagrant project is not updated yet for OCNE 2 and deploys the latest OCNE 1.* release. - -This Vagrant project will deploy and configure the following components: - -- One or more control plane nodes (one by default, 3 in HA mode) -- One or more worker nodes (2 by default) -- An optional operator node for the Oracle Cloud Native Environment -Platform API Server and Platform CLI tool (default is to install these -components on the first control plane node) - -If you enable multiple control plane nodes, an operator node is automatically deployed -to provide egress routing for the cluster. - -All control plane and worker nodes will have the Oracle Cloud Native -Environment Platform Agent installed and configured to communicate with the -Platform API Server on the operator node. - -The installation includes the Kubernetes module for Oracle Cloud -Native Environment which deploys Kubernetes [1.25.7](https://docs.oracle.com/en/operating-systems/olcne/1.6/relnotes/components.html#components) -configured to use the CRI-O runtime interface. Two runtime engines are installed, -[runC](https://docs.oracle.com/en/operating-systems/olcne/1.6/runtimes/runc.html#runc) -and (now deprecated) [Kata](https://docs.oracle.com/en/operating-systems/olcne/1.6/runtimes/kata.html#kata) Containers. - -You may set your cluster networking to Flannel (default), Calico or Multus on top of either Calico or Flannel. - -You may optionally enable the deployment of Istio, MetalLB or Gluster (deprecated) -modules. Note that Helm is now installed automatically. - -_Note:_ Kata Containers (now deprecated) require Intel hardware virtualization support and -will not work in a VirtualBox guest until nested virtualization support is -released for Intel CPUs. - -## Prerequisites - -1. Read the [prerequisites in the top level README](../README.md#prerequisites) -to set up Vagrant with either VirtualBox or KVM -2. [vagrant-env](https://github.com/gosuri/vagrant-env) plugin is optional but -makes configuration much easier - -## Quick start - -1. Clone this repository `git clone https://github.com/oracle/vagrant-projects` -2. Change into the `vagrant-projects/OCNE` directory -3. Run `vagrant up` - -Your Oracle Cloud Native Environment is ready! - -From any control plane node (e.g. controlplane1) you can check the status of the cluster (as -the `vagrant` user). E.g.: - -- `kubectl cluster-info` -- `kubectl get nodes` -- `kubectl get pods --namespace=kube-system` - -## Accessing the Kubernetes Dashboard - -By default, the Kubernetes Dashboord does not allow non-HTTPS connections from -any source except `localhost`/`127.0.0.1`. If you want to be able to connect -to the Dashboard from a browser on your Vagrant host, you will need to set -`BIND_PROXY` to `true` in your `.env.local` file. - -To access the Kubernetes Dashboard, remember to use `localhost` or `127.0.0.1` -in the URL, i.e. . -To obtain token from any Control plane node, you may run: `kubectl -n kubernetes-dashboard get secret -o=jsonpath='{.items[?(@.metadata.annotations.kubernetes\.io/service-account\.name=="kubernetes-dashboard")].data.token}' | base64 --decode` - -## About the `Vagrantfile` - -The VMs communicate via a private network: - -- Controller node IP: 192.168.99.100 (if `STANDALONE_OPERATOR=true`) -- Control plane node _i_: 192.168.99.(100+ _i_ ) / controlplane *_i_* .vagrant.vm -- Worker node _i_: 192.168.99.(110+ _i_ ) / worker *_i_* .vagrant.vm -- Control plane Virtual IP: 192.168.99.99 (if `MULTI_CONTROL_PLANE=true`) -- LoadBalancer IPs: 192.168.99.240 - 192.168.99.250 (if `DEPLOY_METALLB=true`) - -## Configuration - -The `Vagrantfile` can be used _as-is_; there are a couple of parameters you -can set to tailor the installation to your needs. - -### How to configure - -There are several ways to set parameters: - -1. Update the `Vagrantfile`. This is straightforward; the downside is that you -will lose changes when you update this repository. -2. Use environment variables. Might be difficult to remember the parameters -used when the VM was instantiated. -3. Use the `.env`/`.env.local` files (requires -[vagrant-env](https://github.com/gosuri/vagrant-env) plugin). Configure -your cluster by editing the `.env` file; or better copy `.env` to `.env.local` -and edit the latter one, it won't be overridden when you update this repository -and it won't mark your git tree as changed (you won't accidentally commit your -local configuration!) - -Parameters are considered in the following order (first one wins): - -1. Environment variables -2. `.env.local` (if [vagrant-env](https://github.com/gosuri/vagrant-env) plugin -is installed) -3. `.env` (if [vagrant-env](https://github.com/gosuri/vagrant-env) plugin -is installed) -4. `Vagrantfile` definitions - -### VM parameters - -- `VERBOSE` (default: `false`): verbose output during VM deployment. -- `WORKER_CPUS` (default: `1`): Provision Worker Node with 1 vCPU. -- `WORKER_MEMORY` (default: `1024`): Provision Worker Node with 1GB memory. -- `CONTROL_PLANE_CPUS` (default: `2`): At least 2 vCPUS are required for Control plane Nodes. -- `CONTROL_PLANE_MEMORY` (default: `2048`): At least 1700MB are required for Control plane Nodes. -- `OPERATOR_CPUS` (default: `1`): Only applicable if `STANDALONE_OPERATOR=true` or `MULTI_CONTROL_PLANE=true`. -- `OPERATOR_MEMORY` (default: `1024`): Only applicable if `STANDALONE_OPERATOR=true` or `MULTI_CONTROL_PLANE=true`. -- `VB_GROUP` (default: `OCNE`): group all VirtualBox VMs under this label. -- `EXTRA_DISK` (default: `false`): Creates an extra disk (`/dev/sdb`) on Worker nodes that can be used for GlusterFS for Kubernetes Persistent Volumes - -### Cluster parameters - -- `STANDALONE_OPERATOR` (default: `false` unless `MULTI_CONTROL_PLANE=true`): create -a separate VM for the operator node -- default is to install the operator -components on the (first) control plane node. -- `MULTI_CONTROL_PLANE` (default: `false`): multi-control-plane setup. Deploy 3 control planes in -HA mode. -- `NB_WORKERS` (default: `2`): number of worker nodes to provision. -At least one worker node is required. -- `BIND_PROXY` (default: `false`): bind the kubectl proxy port (8001) from the -(first) control plane to the Vagrant host. This is required if you want to access the -Kubernetes Dashboard from a browser on your host. -__Note__: you only need this if you want to expose the kubectl proxy to other -hosts in your network. -- `DEPLOY_CALICO` (default: `false`): deploys the Calico networking module with custom configuration `calico-config.yaml`. -- `DEPLOY_MULTUS` (default: `false`): deploys the Multus networking module with custom configuration `multus-config.yaml`. -- `DEPLOY_HELM` (default: `false`): deploys the Helm module (deprecated). -- `DEPLOY_ISTIO` (default: `false`): deploys the Istio modules. -- `DEPLOY_METALLB` (default: `false`): deploys the MetalLB module. -- `DEPLOY_GLUSTER` (default: `false`): deploys the Gluster module (deprecated). -__Note__: if `NB_WORKERS` is less than `3`, the `hyperconverged` `storageclass` -is patched to adjust the number of Gluster replicas accordingly. -__Note__: This provisioning script also installs Heketi on the operator node. - -### Repositories - -- `YUM_REPO` (default: none): additional yum repository to consider -(e.g. local repo) -- `OCNE_DEV` (default: `false`): whether to enable the Oracle Cloud -Native Environment developer channel. -- `REGISTRY_OCNE` (default: `container-registry.oracle.com/olcne`): Container -registry for Oracle Cloud Native Environment images. - -For performance reasons, we recommend using the closest Oracle Container Registry mirror to your region. A list of available regions can be found on the [Regions and Availability Domains](https://docs.cloud.oracle.com/en-us/iaas/Content/General/Concepts/regions.htm) page of the Oracle Cloud Infrastructure documentation. - -To specify an Oracle Container Registry mirror, either edit the `Vagrantfile` or install the vagrant-env plugin and create a `.env.local` file that specifies the mirror. - -The following syntax can be used to specify a mirror: - -- `container-registry-.oracle.com`, e.g. `container-registry-sydney.oracle.com/olcne` -- `container-registry-.oracle.com`, e.g. `container-registry-ap-sydney-1.oracle.com/olcne` -- `container-registry-.oracle.com`, e.g. `container-registry-syd.oracle.com/olcne` - - All regions are available at - -### Advanced Parameters - -Danger zone! -Mainly used for development. - -- `NB_CONTROL_PLANES` (default: none): override number of control planes to deploy. Requires `MULTI_CONTROL_PLANE=true` to function properly. -- `SUBNET` (default: `192.168.99`): Set the VM provider host-only / private network subnet. -- `UPDATE_OS` (default: false): Runs `dnf -y update` on the VM. - -## Optional plugins - -When installed, this `Vagrantfile` will make use of the following third party Vagrant plugins: - -- [vagrant-env](https://github.com/gosuri/vagrant-env): loads environment -variables from .env files; -- [vagrant-hosts](https://github.com/oscar-stack/vagrant-hosts): maintains -`/etc/hosts` for the guest VMs; -- [vagrant-proxyconf](https://github.com/tmatilai/vagrant-proxyconf): set -proxies in the guest VMs if you need to access the Internet through proxy. See -plugin documentation for the configuration. - -To install Vagrant plugins run: - -```shell -vagrant plugin install ... -``` - -## Product Documentation - -- [Oracle Cloud Native Environment: Getting Started](https://docs.oracle.com/en/operating-systems/olcne/1.6/start/) -- [Oracle Cloud Native Environment: Container Orchestration](https://docs.oracle.com/en/operating-systems/olcne/1.6/orchestration/) -- [Oracle Cloud Native Environment: Container Runtimes](https://docs.oracle.com/en/operating-systems/olcne/1.6/runtimes/) -- [Oracle Cloud Native Environment: Platform CLI](https://docs.oracle.com/en/operating-systems/olcne/1.6/olcnectl/) - -## Feedback - -Please provide feedback of any kind via GitHub issues on this repository. - -## Contributing - -See [CONTRIBUTING](../CONTRIBUTING.md) for details. diff --git a/OCNE/Vagrantfile b/OCNE/Vagrantfile deleted file mode 100644 index 6b3b517c..00000000 --- a/OCNE/Vagrantfile +++ /dev/null @@ -1,349 +0,0 @@ -# -# Vagrantfile for Oracle Cloud Native Environment -# -# Copyright (c) 2019, 2022 Oracle and/or its affiliates. -# Licensed under the Universal Permissive License v 1.0 as shown at -# https://oss.oracle.com/licenses/upl. -# -# Description: Deploys an Oracle Cloud Native Environment -# -# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS HEADER. -# - -# -*- mode: ruby -*- -# vi: set ft=ruby : - -# This Vagrantfile creates an Oracle Cloud Native Environment and -# deploys the Kubernetes module to the control plane and worker nodes. -# VMs communicate via a private network using subnet 192.168.99.* (by default): -# - HA Control plane IP: 192.168.99.99 (Virtual IP, when in HA mode) -# - Operator : 192.168.99.100 (Optional, none by default) -# - Control plane i : 192.168.99.(100+i) (1 by default, 3 in HA mode) -# - Worker i : 192.168.99.(110+i) (2 by default) -# - MetalLB Pool : 192.168.99.240 - 192.168.99.250 -# -# Optional plugins: -# vagrant-hosts (maintains /etc/hosts for the VMs) -# vagrant-env (use .env files for configuration) -# vagrant-proxyconf (if you don't have direct access to the Internet) -# see https://github.com/tmatilai/vagrant-proxyconf for configuration -# - -# Required for the Disks feature -Vagrant.require_version ">= 2.2.8" -ENV['VAGRANT_EXPERIMENTAL'] = 'disks' - -# Vagrantfile API/syntax version. Don't touch unless you know what you're doing! -VAGRANTFILE_API_VERSION = "2" -ENV['VAGRANT_NO_PARALLEL'] = 'yes' - -# Box metadata location and box name -BOX_URL = "https://oracle.github.io/vagrant-projects/boxes" -BOX_NAME = "oraclelinux/8" - -# Define constants -Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| - # Use vagrant-env plugin if available - if Vagrant.has_plugin?("vagrant-env") - config.env.load(".env.local", ".env") # enable the plugin - end - - # Default Private Network Subnet - SUBNET = default_s('SUBNET', '192.168.99') - - # vCPUS and Memory for the VMs - OPERATOR_CPUS = default_i('OPERATOR_CPUS', 1) - OPERATOR_MEMORY = default_i('OPERATOR_MEMORY', 1024) - CONTROL_PLANE_CPUS = default_i('CONTROL_PLANE_CPUS', 2) - CONTROL_PLANE_MEMORY = default_i('CONTROL_PLANE_MEMORY', 2048) - WORKER_CPUS = default_i('WORKER_CPUS', 1) - WORKER_MEMORY = default_i('WORKER_MEMORY', 1024) - - # Group VirtualBox containers - VB_GROUP = default_s('VB_GROUP', 'OCNE') - - # Multi-control-plane setup. Deploy 3 control planes in HA mode. - MULTI_CONTROL_PLANE = default_b('MULTI_CONTROL_PLANE', false) - - # Separate operator node for the Oracle Cloud Native Environment - # Platform API Server and Platform Agent (default is to install the - # components on the (first) control plane node - # - # If multi-control-plane is enabled, the standalone operator is automatically - # enabled for routing purposes - if MULTI_CONTROL_PLANE - STANDALONE_OPERATOR = true - else - STANDALONE_OPERATOR = default_b('STANDALONE_OPERATOR', false) - end - - # Creates an extra disk (/dev/sdb) so it can be used as a - # Gluster Storage for Kubernetes Persistent Volumes - EXTRA_DISK = default_b('EXTRA_DISK', false) - - # Override number of control planes to deploy - # This should not be changed -- for development purpose - NB_CONTROL_PLANES = default_i('NB_CONTROL_PLANES', MULTI_CONTROL_PLANE ? 3 : 1) - - # Number of worker nodes to provision - NB_WORKERS = default_i('NB_WORKERS', 2) - - # Bind the kubectl proxy from the (first) control plane to the vagrant host - BIND_PROXY = default_b('BIND_PROXY', false) - - # Additional yum channel to consider (e.g. local repo) - YUM_REPO = default_s('YUM_REPO', '') - - # Add Oracle Cloud Native Environment developer channel - OCNE_DEV = default_b('OCNE_DEV', false) - - # Set the default OCNE_ENV_NAME and OCNE_CLUSTER_NAMEs - OCNE_ENV_NAME = default_s('OCNE_ENV_NAME', 'ocne-env') - OCNE_CLUSTER_NAME = default_s('OCNE_CLUSTER_NAME', 'ocne-cluster') - - # Container registry for Oracle Cloud Native Environment images - # You can use registry mirrors in a region close to you. - # Check the README.md file for more details. - REGISTRY_OCNE = default_s('REGISTRY_OCNE', 'container-registry.oracle.com/olcne') - - # Deploy Calico networking module? - DEPLOY_CALICO = default_b('DEPLOY_CALICO', false) - - # Deploy Multus networking module? - DEPLOY_MULTUS = default_b('DEPLOY_MULTUS', false) - - # Deploy Istio? - DEPLOY_ISTIO = default_b('DEPLOY_ISTIO', false) - - # Deploy MetalLB? - DEPLOY_METALLB = default_b('DEPLOY_METALLB', false) - - # Deploy Gluster? (deprecated) - DEPLOY_GLUSTER = default_b('DEPLOY_GLUSTER', false) - - # Deploy Helm? (deprecated) - DEPLOY_HELM = default_b('DEPLOY_HELM', false) - - CALICO_MODULE_NAME = default_s('CALICO_MODULE_NAME', 'ocne-calico') - MULTUS_MODULE_NAME = default_s('MULTUS_MODULE_NAME', 'ocne-multus') - HELM_MODULE_NAME = default_s('HELM_MODULE_NAME', 'ocne-helm') - ISTIO_MODULE_NAME = default_s('ISTIO_MODULE_NAME', 'ocne-istio') - METALLB_MODULE_NAME = default_s('METALLB_MODULE_NAME', 'ocne-metallb') - GLUSTER_MODULE_NAME = default_s('GLUSTER_MODULE_NAME', 'ocne-gluster') - - # Update Base OS - UPDATE_OS = default_b('UPDATE_OS', false) - - # Verbose console - VERBOSE = default_b('VERBOSE', false) -end - -# Convenience methods -def default_s(key, default) - ENV[key] && ! ENV[key].empty? ? ENV[key] : default -end - -def default_i(key, default) - default_s(key, default).to_i -end - -def default_b(key, default) - default_s(key, default).to_s.downcase == "true" -end - -def ensure_scheme(url) - (url =~ /.*:\/\// ? "" : "http://") + url -end - -def update_os(vm) - if UPDATE_OS - vm.provision :shell, - inline: "dnf -y update", - privileged: true, - reboot: true - end -end - -def provision_vm(vm, vm_args) - args = vm_args.clone - args.push("--ocne-environment-name", OCNE_ENV_NAME) - args.push("--ocne-cluster-name", OCNE_CLUSTER_NAME) - args.push("--multi-control-plane") if MULTI_CONTROL_PLANE - args.push("--repo", YUM_REPO) unless YUM_REPO == "" - args.push("--ocne-dev") if OCNE_DEV - args.push("--with-calico") if DEPLOY_CALICO - args.push("--calico-module-name", CALICO_MODULE_NAME) if DEPLOY_CALICO - args.push("--with-multus") if DEPLOY_MULTUS - args.push("--multus-module-name", MULTUS_MODULE_NAME) if DEPLOY_MULTUS - args.push("--with-helm") if DEPLOY_HELM - args.push("--helm-module-name", HELM_MODULE_NAME) if DEPLOY_HELM - args.push("--with-istio") if DEPLOY_ISTIO - args.push("--istio-module-name", ISTIO_MODULE_NAME) if DEPLOY_ISTIO - args.push("--with-metallb") if DEPLOY_METALLB - args.push("--metallb-module-name", METALLB_MODULE_NAME) if DEPLOY_METALLB - args.push("--with-gluster") if DEPLOY_GLUSTER - args.push("--gluster-module-name", GLUSTER_MODULE_NAME) if DEPLOY_GLUSTER - args.push("--registry-ocne", REGISTRY_OCNE) if REGISTRY_OCNE - args.push("--verbose") if VERBOSE - vm.provision "shell", - path: "scripts/provision.sh", - args: args, - privileged: false -end - -Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| - - config.vm.box = BOX_NAME - config.vm.box_url = "#{BOX_URL}/#{BOX_NAME}.json" - # config.vm.box_version = "8.6.359" - - # If we use the vagrant-proxyconf plugin, we should not proxy k8s/local IPs - # Unfortunately we can't use CIDR with no_proxy, so we have to enumerate and - # 'blacklist' *all* IPs - if Vagrant.has_plugin?("vagrant-proxyconf") - has_proxy = false - ["http_proxy", "HTTP_PROXY"].each do |proxy_var| - if proxy = ENV[proxy_var] - puts "HTTP proxy: " + proxy - config.proxy.http = ensure_scheme(proxy) - has_proxy = true - break - end - end - - ["https_proxy", "HTTPS_PROXY"].each do |proxy_var| - if proxy = ENV[proxy_var] - puts "HTTPS proxy: " + proxy - config.proxy.https = ensure_scheme(proxy) - has_proxy = true - break - end - end - - if has_proxy - # Only consider no_proxy if we have proxies defined. - no_proxy = "" - ["no_proxy", "NO_PROXY"].each do |proxy_var| - if ENV[proxy_var] - no_proxy = ENV[proxy_var] - puts "No proxy: " + no_proxy - no_proxy += "," - break - end - end - config.proxy.no_proxy = no_proxy + "localhost,.vagrant.vm," + (".0"..".255").to_a.join(",") - end - end - - # Provider-specific configuration -- VirtualBox - config.vm.provider :virtualbox do |vb| - vb.linked_clone = false - vb.customize ["modifyvm", :id, "--groups", "/" + VB_GROUP] - vb.customize ["modifyvm", :id, "--nested-hw-virt", "on"] - end - config.vm.provider :libvirt do |lv| - lv.nested = true - end - - # Workers provisioning - workers = "" - (1..NB_WORKERS).each do |i| - config.vm.define "worker#{i}" do |worker| - worker.vm.hostname = "worker#{i}.vagrant.vm" - ip = 110 + i - ip_addr = "#{SUBNET}.#{ip}" - workers += "#{ip_addr}," -# worker.vm.network :private_network, nic_type: "virtio", ip: ip_addr, nm_controlled: "yes" - worker.vm.network :private_network, nic_type: "virtio", ip: ip_addr - if Vagrant.has_plugin?("vagrant-hosts") - worker.vm.provision :hosts, :sync_hosts => true, :add_localhost_hostnames => false - end - worker.vm.provider :virtualbox do |vb, override| - vb.name = "worker#{i}" - vb.memory = WORKER_MEMORY - vb.cpus = WORKER_CPUS - if EXTRA_DISK - override.vm.disk :disk, size: '16GB', name: 'extra_disk' - end - end - config.vm.provider :libvirt do |lv| - lv.memory = WORKER_MEMORY - lv.cpus = WORKER_CPUS - if EXTRA_DISK - lv.storage :file, :size => '16G', :type => 'qcow2' - end - end - # Update OS if UPDATE_OS=true - update_os(worker.vm) - # Provisioning: Worker Node - provision_vm(worker.vm, ["--worker"]) - end - end - - # Control plane provisioning - control_planes = "" - NB_CONTROL_PLANES.downto(1) do |i| - config.vm.define "controlplane#{i}" do |control_plane| - control_plane.vm.hostname = "controlplane#{i}.vagrant.vm" - ip = 100 + i - ip_addr = "#{SUBNET}.#{ip}" - control_planes += "#{ip_addr}," - control_plane.vm.network :private_network, nic_type: "virtio", ip: ip_addr - if Vagrant.has_plugin?("vagrant-hosts") - control_plane.vm.provision :hosts, :sync_hosts => true, :add_localhost_hostnames => false - end - control_plane.vm.provider :virtualbox do |vb| - vb.name = "control_plane#{i}" - vb.memory = CONTROL_PLANE_MEMORY - vb.cpus = CONTROL_PLANE_CPUS - end - config.vm.provider :libvirt do |lv| - lv.memory = CONTROL_PLANE_MEMORY - lv.cpus = CONTROL_PLANE_CPUS - end - if BIND_PROXY && i == 1 - # Bind kubectl proxy port - control_plane.vm.network :forwarded_port, guest: 8001, host: 8001 - end - # Update OS if UPDATE_OS=true - update_os(control_plane.vm) - # Provisioning: Control plane Node - args = ["--control-plane"] - if !STANDALONE_OPERATOR && i == 1 - args.push("--operator") - args.push("--subnet", SUBNET) - args.push("--workers", workers.chop) - args.push("--control-planes", control_planes.chop) - end - provision_vm(control_plane.vm, args) - end - end - - # Operator node, if STANDALONE_OPERATOR=true - if STANDALONE_OPERATOR - config.vm.define "operator" do |operator| - operator.vm.hostname = "operator.vagrant.vm" - operator.vm.network :private_network, nic_type: "virtio", ip: "#{SUBNET}.100" - if Vagrant.has_plugin?("vagrant-hosts") - operator.vm.provision :hosts, :sync_hosts => true, :add_localhost_hostnames => false - end - operator.vm.provider :virtualbox do |vb| - vb.name = "operator" - vb.memory = OPERATOR_MEMORY - vb.cpus = OPERATOR_CPUS - end - config.vm.provider :libvirt do |lv| - lv.memory = OPERATOR_MEMORY - lv.cpus = OPERATOR_CPUS - end - # Update OS if UPDATE_OS=true - update_os(operator.vm) - # Provisioning: Operator Node - args = ["--operator"] - args.push("--subnet", SUBNET) - args.push("--workers", workers.chop) - args.push("--control-planes", control_planes.chop) - provision_vm(operator.vm, args) - end - end -end diff --git a/OCNE/scripts/provision.sh b/OCNE/scripts/provision.sh deleted file mode 100755 index 0566db23..00000000 --- a/OCNE/scripts/provision.sh +++ /dev/null @@ -1,942 +0,0 @@ -#!/bin/bash -# -# Provision Oracle Cloud Native Environment nodes -# -# Copyright (c) 2019, 2022 Oracle and/or its affiliates. -# Licensed under the Universal Permissive License v 1.0 as shown at -# https://oss.oracle.com/licenses/upl. -# -# Description: Installs the Oracle Cloud Native Environment packages, -# configures all prerequisites and deploys the Kubernetes module. -# -# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS HEADER. -# - -####################################### -# Convenience function used to limit output during provisioning -# Exit on error -# Prepend any command with "echo_do" -# Caveats: -# - Quoted parameters need to be quoted 2 times -# E.g.: echo_do ls "'a b'" -# - Statements with redirects need to be evaluated twice -# E.g.: echo_do eval "ls >x" -# Globals: -# VERBOSE -# Arguments: -# Command to run -# Returns: -# None -####################################### -echo_do() { - local tmp_file - local ret_code - - [[ -n "${VERBOSE}" ]] && echo " $*" - tmp_file=$(mktemp /var/tmp/cmd_XXXXX.log) - eval "$@" > "${tmp_file}" 2>&1 - ret_code=$? - if [[ ${ret_code} -ne 0 ]]; then - [[ -z "${VERBOSE}" ]] && echo "$@" - echo "Returned a non-zero code: ${ret_code}" >&2 - echo "Last output lines:" >&2 - tail -5 "${tmp_file}" >&2 - echo "See ${tmp_file} for details" >&2 - exit ${ret_code} - fi - rm "${tmp_file}" -} - -####################################### -# Just print a message -# Globals: -# None -# Arguments: -# Text to be printed -# Returns: -# None -####################################### -msg() { - echo "===== ${*} =====" -} - -####################################### -# Parse arguments -# Exit on error. -# Globals: -# OCNE_DEV CONTROL_PLANE CONTROL_PLANES WORKER WORKERS -# OPERATOR MULTI_CONTROL_PLANE REGISTRY_OCNE VERBOSE EXTRA_REPO -# Arguments: -# Command line -# Returns: -# None -####################################### -parse_args() { - OCNE_CLUSTER_NAME='' OCNE_ENV_NAME='' OCNE_DEV=0 REGISTRY_OCNE='' - OPERATOR=0 MULTI_CONTROL_PLANE=0 CONTROL_PLANE=0 CONTROL_PLANES='' WORKER=0 WORKERS='' - VERBOSE=0 SUBNET='' EXTRA_REPO='' - DEPLOY_CALICO=0 CALICO_MODULE_NAME='' DEPLOY_MULTUS=0 MULTUS_MODULE_NAME='' - DEPLOY_HELM=0 HELM_MODULE_NAME='' DEPLOY_ISTIO=0 ISTIO_MODULE_NAME='' - DEPLOY_METALLB=0 METALLB_MODULE_NAME='' DEPLOY_GLUSTER=0 GLUSTER_MODULE_NAME='' - - while [[ $# -gt 0 ]]; do - case "$1" in - "--control-plane") - CONTROL_PLANE=1 - shift - ;; - "--worker") - WORKER=1 - shift - ;; - "--operator") - OPERATOR=1 - shift - ;; - "--multi-control-plane") - MULTI_CONTROL_PLANE=1 - shift - ;; - "--ocne-dev") - OCNE_DEV=1 - shift - ;; - "--ocne-environment-name") - if [[ $# -lt 2 ]]; then - echo "Missing parameter for --ocne-environment-name" >&2 - exit 1 - fi - OCNE_ENV_NAME="$2" - shift; shift; - ;; - "--ocne-cluster-name") - if [[ $# -lt 2 ]]; then - echo "Missing parameter for --ocne-cluster-name" >&2 - exit 1 - fi - OCNE_CLUSTER_NAME="$2" - shift; shift; - ;; - "--repo") - if [[ $# -lt 2 ]]; then - echo "Missing parameter for --repo" >&2 - exit 1 - fi - EXTRA_REPO="$2" - shift; shift - ;; - "--registry-ocne") - if [[ $# -lt 2 ]]; then - echo "Missing parameter for --registry-ocne" >&2 - exit 1 - fi - REGISTRY_OCNE="$2" - shift; shift - ;; - "--control-planes") - if [[ $# -lt 2 ]]; then - echo "Missing parameter for --control-planes" >&2 - exit 1 - fi - CONTROL_PLANES="$2" - shift; shift - ;; - "--workers") - if [[ $# -lt 2 ]]; then - echo "Missing parameter for --workers" >&2 - exit 1 - fi - WORKERS="$2" - shift; shift - ;; - "--with-calico") - DEPLOY_CALICO=1 - shift - ;; - "--calico-module-name") - if [[ $# -lt 2 ]]; then - echo "Missing parameter for --calico-module-name" >&2 - exit 1 - fi - CALICO_MODULE_NAME="$2" - shift; shift - ;; - "--with-multus") - DEPLOY_MULTUS=1 - shift - ;; - "--multus-module-name") - if [[ $# -lt 2 ]]; then - echo "Missing parameter for --multus-module-name" >&2 - exit 1 - fi - MULTUS_MODULE_NAME="$2" - shift; shift - ;; - "--with-helm") - DEPLOY_HELM=1 - shift - ;; - "--helm-module-name") - if [[ $# -lt 2 ]]; then - echo "Missing parameter for --helm-module-name" >&2 - exit 1 - fi - HELM_MODULE_NAME="$2" - shift; shift - ;; - "--with-istio") - DEPLOY_ISTIO=1 - shift - ;; - "--istio-module-name") - if [[ $# -lt 2 ]]; then - echo "Missing parameter for --istio-module-name" >&2 - exit 1 - fi - ISTIO_MODULE_NAME="$2" - shift; shift - ;; - "--with-metallb") - DEPLOY_METALLB=1 - shift - ;; - "--metallb-module-name") - if [[ $# -lt 2 ]]; then - echo "Missing parameter for --metallb-module-name" >&2 - exit 1 - fi - METALLB_MODULE_NAME="$2" - shift; shift - ;; - "--with-gluster") - DEPLOY_GLUSTER=1 - shift - ;; - "--gluster-module-name") - if [[ $# -lt 2 ]]; then - echo "Missing parameter for --gluster-module-name" >&2 - exit 1 - fi - GLUSTER_MODULE_NAME="$2" - shift; shift - ;; - "--subnet") - if [[ $# -lt 2 ]]; then - echo "Missing parameter for --subnet" >&2 - exit 1 - fi - SUBNET="$2" - shift; shift; - ;; - "--verbose") - VERBOSE=1 - shift - ;; - *) - echo "Invalid parameter: $1" >&2 - exit 1 - ;; - esac - done - - readonly OCNE_CLUSTER_NAME OCNE_ENV_NAME OCNE_DEV REGISTRY_OCNE - readonly OPERATOR MULTI_CONTROL_PLANE CONTROL_PLANE CONTROL_PLANES WORKER WORKERS - readonly VERBOSE EXTRA_REPO - readonly DEPLOY_CALICO CALICO_MODULE_NAME - readonly DEPLOY_MULTUS MULTUS_MODULE_NAME - readonly DEPLOY_HELM HELM_MODULE_NAME - readonly DEPLOY_ISTIO ISTIO_MODULE_NAME - readonly DEPLOY_METALLB METALLB_MODULE_NAME - readonly DEPLOY_GLUSTER GLUSTER_MODULE_NAME -} - -####################################### -# Configure repos for the installation -# Globals: -# EXTRA_REPO -# OCNE_DEV -# Arguments: -# None -# Returns: -# None -####################################### -setup_repos() { - msg "Configure dnf repos for Oracle Cloud Native Environment" - - # Workaround for ol8_developer channels not available bug - echo_do sudo dnf install -y oraclelinux-developer-release-el8 - - if [[ ${OPERATOR} == 1 ]]; then - echo_do sudo dnf install -y oracle-olcne-release-el8 - echo_do sudo dnf config-manager --enable ol8_olcne19 ol8_addons ol8_baseos_latest ol8_appstream ol8_kvm_appstream ol8_UEKR7 - echo_do sudo dnf config-manager --disable ol8_olcne18 ol8_olcne17 ol8_olcne16 ol8_olcne15 ol8_olcne14 ol8_olcne13 ol8_olcne12 - fi - - # Optional extra repo - if [[ -n ${EXTRA_REPO} ]]; then echo_do sudo dnf config-manager --add-repo "${EXTRA_REPO}"; fi - - # Enable OCNE developer channel - if [[ ${OCNE_DEV} == 1 ]]; then echo_do sudo dnf config-manager --enable ol8_developer_olcne; fi -} - -####################################### -# Configure prerequisites -# Globals: -# None -# Arguments: -# None -# Returns: -# None -####################################### -prerequisites() { - - if [[ ${DEPLOY_CALICO} == 1 ]]; then - msg "Installing kernel-uek-modules for calico" - echo_do sudo dnf install -y kernel-uek-modules-$(uname -r) - fi - - if [[ ${DEPLOY_GLUSTER} == 1 ]]; then - if [[ ${WORKER} == 1 ]]; then - msg "Installing the GlusterFS Server on Worker node" - echo_do sudo dnf install -y oracle-gluster-release-el8 - echo_do sudo dnf config-manager --enable ol8_gluster_appstream - echo_do sudo dnf module enable -y glusterfs - echo_do sudo dnf install -y @glusterfs/server - # Enable TLS / Management Encryption - # https://docs.oracle.com/en/operating-systems/oracle-linux/gluster-storage/gluster-install-upgrade.html#gluster-tls - msg "Enable GlusterFS Transport Layer Security (TLS) for Management Encryption" - echo_do sudo openssl genrsa -out /etc/ssl/glusterfs.key 2048 - echo_do sudo openssl req -new -x509 -days 365 -key /etc/ssl/glusterfs.key -out /etc/ssl/glusterfs.pem -subj '/CN=`hostname -f`' - echo_do eval "cat /etc/ssl/glusterfs.pem >> /vagrant/glusterfs.ca" - echo_do sudo touch /var/lib/glusterd/secure-access - echo_do sudo systemctl enable --now glusterd.service - echo_do sudo firewall-cmd --add-service=glusterfs --permanent - fi - - if [[ ${OPERATOR} == 1 ]]; then - if [[ -f "/vagrant/glusterfs.ca" ]]; then - msg "Distributing GlusterFS Certificate Authority's (CA) certificates" - for node in ${WORKERS//,/ }; do - echo_do ssh -i /vagrant/id_rsa -o "UserKnownHostsFile=/vagrant/known_hosts" "${node}" "sudo cp /vagrant/glusterfs.ca /etc/ssl/glusterfs.ca" - done - echo_do "rm -f /vagrant/glusterfs.ca" - fi - - msg "Installing the Heketi Server & CLI on Operator node" - echo_do sudo dnf install -y oracle-gluster-release-el8 - echo_do sudo dnf config-manager --enable ol8_gluster_appstream - echo_do sudo dnf module enable -y glusterfs - echo_do sudo dnf install -y heketi heketi-client - if [[ ${MASTER} == 0 ]]; then - # Standalone operator - echo_do sudo firewall-cmd --add-port=8080/tcp --permanent - fi - msg "Modifying the default /etc/heketi/heketi.json onto /vagrant/heketi.json" - echo_do sudo dnf install -y jq - contents="$(jq '.use_auth=true|.jwt.admin.key="secret"|.glusterfs.executor="ssh"|.glusterfs.sshexec.keyfile="/etc/heketi/vagrant_key"|.glusterfs.sshexec.user="vagrant"|.glusterfs.sshexec.sudo=true|del(.glusterfs.sshexec.port)|del(.glusterfs.sshexec.fstab)|.glusterfs.loglevel="info"' /etc/heketi/heketi.json)" && echo -E "${contents}" > /vagrant/heketi.json - echo_do sudo cp /vagrant/heketi.json /etc/heketi/heketi.json - echo_do rm -f /vagrant/heketi.json - # SSH Key *MUST* be in PEM format! Heketi would reject it otherwise. - msg "Copying the Vagrant SSH Key. Must be in PEM format!" - echo_do sudo cp /vagrant/id_rsa /etc/heketi/vagrant_key - # Fix default permission which exposes the secret /etc/heketi/heketi.json - echo_do sudo chmod 0600 /etc/heketi/vagrant_key /etc/heketi/heketi.json - echo_do sudo chown -R heketi: /etc/heketi - # Enable Heketi - echo_do sudo systemctl enable --now heketi.service - # Test Heketi - msg "Waiting to Heketi service to become ready" - echo_do curl --retry-connrefused --retry 10 --retry-delay 5 127.0.0.1:8080/hello - # Heketi ready - msg "Creating Gluster Topology file /etc/heketi/topology-ocne.json" - # https://github.com/heketi/heketi/blob/master/docs/admin/topology.md - jq -R '{clusters:[{nodes:(./","|map({node:{hostnames:{manage:[.],storage:[.]},zone:1},devices:[{name:"/dev/sdb",destroydata:false}]}))}]}' <<< "${WORKERS}" > /vagrant/topology-ocne.json - echo_do sudo cp /vagrant/topology-ocne.json /etc/heketi/topology-ocne.json - echo_do sudo chown heketi: /etc/heketi/topology-ocne.json - msg "Loading Gluster Cluster Topology with Heketi" - # export HEKETI_CLI_USER=admin; export HEKETI_CLI_KEY=secret - echo_do heketi-cli --user=admin --secret=secret topology load --json=/etc/heketi/topology-ocne.json - echo_do rm -f /vagrant/topology-ocne.json - fi - fi - - -} - -####################################### -# Clean up private network interface -# Globals: -# None -# Arguments: -# None -# Returns: -# None -####################################### -clean_networking() { - msg "Removing extra NetworkManager connection" - nmcli -f GENERAL.STATE con show "Wired connection 1" && sudo nmcli con del "Wired connection 1" -} - -####################################### -# Configure passwordless ssh between nodes -# Globals: -# OPERATOR -# Arguments: -# None -# Returns: -# None -####################################### -passwordless_ssh() { - msg "Allow passwordless ssh between VMs" - # Generate common key - if [[ ! -f /vagrant/id_rsa && ! -f /vagrant/id_rsa ]]; then - msg "Generating shared SSH keypair in PEM format" - echo_do ssh-keygen -m PEM -t rsa -f /vagrant/id_rsa -q -N "''" -C "'vagrant@ocne'" - fi - # Generate known_hosts - if [[ ! -f /vagrant/known_hosts ]]; then - msg "Generating shared SSH Known Hosts file" - echo_do cp /dev/null /vagrant/known_hosts - fi - # Install private key & set permissions - echo_do "[ -d ~/.ssh ] || ( mkdir ~/.ssh && chmod 0700 ~/.ssh )" - echo_do "[ -f ~/.ssh/id_rsa ] || ( cp /vagrant/id_rsa ~/.ssh && chmod 0600 ~/.ssh/id_rsa )" - # Authorise passwordless ssh - echo_do "[ -f ~/.ssh/id_rsa.pub ] || ( cp /vagrant/id_rsa.pub ~/.ssh && echo_do chmod 0644 ~/.ssh/id_rsa.pub && cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys )" - # SSH Host Keys. Should really use ssh-keyscan -t ecdsa,ed25519 - echo_do eval '[ -f /etc/ssh/ssh_known_hosts ] || echo "`hostname -s`,`hostname -f`,`hostname -I|sed "s/ $//;s/ /,/g"` `cat /etc/ssh/ssh_host_ed25519_key.pub`" >> /vagrant/known_hosts' - # Last node removes the key - if [[ ${OPERATOR} == 1 ]]; then - if [[ -f /vagrant/id_rsa && -f /vagrant/id_rsa.pub ]]; then - msg "Removing the shared SSH keypair" - echo_do rm -f /vagrant/id_rsa /vagrant/id_rsa.pub - fi - if [[ -f /vagrant/known_hosts ]]; then - msg "Copying SSH Host Keys to allow StrictHostKeyChecking" - echo_do "[ -f /etc/ssh/ssh_known_hosts ] || sudo cp /vagrant/known_hosts /etc/ssh/ssh_known_hosts" - for node in ${CONTROL_PLANES//,/ } ${WORKERS//,/ }; do - echo_do ssh "${node}" "sudo cp /vagrant/known_hosts /etc/ssh/ssh_known_hosts" - done - msg "Removing the shared SSH Known Hosts file" - echo_do rm -f /vagrant/known_hosts - fi - fi -} - -####################################### -# OCNE Quick Install -# Globals: -# CONTROL_PLANES MULTI_CONTROL_PLANE -# OCNE_CLUSTER_NAME OCNE_ENV_NAME -# REGISTRY_OCNE -# Arguments: -# None -# Returns: -# None -####################################### -quick_install_ocne() { - local api_server provision_opts='' - - echo_do sudo dnf install -y olcnectl - - if [[ ${CONTROL_PLANE} == 1 ]]; then - api_server=${CONTROL_PLANES//,*/} - else - api_server=$(ip -f inet addr show eth1| sed -En -e 's/.*inet ([0-9.]+).*/\1/p') - fi - - provision_opts=(--api-server "${api_server}" --control-plane-nodes "${CONTROL_PLANES}" --worker-nodes "${WORKERS}") - provision_opts=("${provision_opts[@]}" --environment-name "${OCNE_ENV_NAME}" --name "${OCNE_CLUSTER_NAME}") - provision_opts=("${provision_opts[@]}" --container-registry "${REGISTRY_OCNE}") - provision_opts=("${provision_opts[@]}" --selinux enforcing) - - if [[ ${MULTI_CONTROL_PLANE} == 1 ]]; then - provision_opts=("${provision_opts[@]}" --virtual-ip "${SUBNET}".99) - fi - - if [[ -n ${HTTP_PROXY} ]]; then - provision_opts=("${provision_opts[@]}" --http-proxy "${HTTP_PROXY}") - provision_opts=("${provision_opts[@]}" --https-proxy "${HTTPS_PROXY}") - provision_opts=("${provision_opts[@]}" --no-proxy "${NO_PROXY}") - fi - - if [[ ${VERBOSE} == 1 ]]; then - provision_opts=("${provision_opts[@]}" --debug) - fi - - msg "Provision the OCNE cluster with quick install" - echo_do olcnectl provision "${provision_opts[@]}" --yes --timeout 20 - - msg "Update config to avoid having to enter the --api-server option in future olcnectl commands" - echo_do olcnectl module instances \ - --api-server "${api_server}:8091" \ - --environment-name "${OCNE_ENV_NAME}" \ - --update-config -} - -####################################### -# Deploy additional modules -# Globals: -# OCNE_CLUSTER_NAME OCNE_ENV_NAME -# DEPLOY_HELM HELM_MODULE_NAME -# DEPLOY_ISTIO ISTIO_MODULE_NAME -# DEPLOY_METALLB METALLB_MODULE_NAME -# DEPLOY_GLUSTER GLUSTER_MODULE_NAME -# REGISTRY_OCNE -# Arguments: -# None -# Returns: -# None -####################################### -deploy_modules() { - local node control_plane_nodes worker_nodes - - msg "Deploying additional modules" - - # Calico networking module - if [[ ${DEPLOY_CALICO} == 1 ]]; then - - # BEGIN WORKAROUND: recreate Kubernetes module until calico can be installed - # with olcnectl provision quick installation - - msg "Workaround: recreate Kubernetes module for Calico pod-network" - - control_plane_nodes="${CONTROL_PLANES//,/:8090,}:8090" - worker_nodes="${WORKERS//,/:8090,}:8090" - - echo_do olcnectl module uninstall \ - --environment-name "${OCNE_ENV_NAME}" \ - --name "${OCNE_CLUSTER_NAME}" - - echo_do olcnectl module create --module kubernetes \ - --environment-name "${OCNE_ENV_NAME}" \ - --name "${OCNE_CLUSTER_NAME}" \ - --container-registry "${REGISTRY_OCNE}" \ - --control-plane-nodes "${control_plane_nodes}" \ - --worker-nodes "${worker_nodes}" \ - --selinux enforcing \ - --pod-network none \ - --pod-network-iface eth1 \ - --restrict-service-externalip false - - echo_do olcnectl module validate \ - --environment-name "${OCNE_ENV_NAME}" \ - --name "${OCNE_CLUSTER_NAME}" - - echo_do olcnectl module install \ - --environment-name "${OCNE_ENV_NAME}" \ - --name "${OCNE_CLUSTER_NAME}" - - # END WORKAROUND - - if ! [ -f /vagrant/calico-config.yaml ]; then - echo_do "cat <<-EOF | tee /vagrant/calico-config.yaml - installation: - cni: - type: Calico - calicoNetwork: - bgp: Disabled - ipPools: - - cidr: 10.244.0.0/16 - encapsulation: VXLAN - nodeAddressAutodetectionV4: - interface: eth1 - registry: container-registry.oracle.com - imagePath: olcne -EOF" - fi - - # Create the Calico networking module - msg "Creating the Calico networking module: ${CALICO_MODULE_NAME}" - echo_do olcnectl module create \ - --environment-name "${OCNE_ENV_NAME}" \ - --module calico \ - --name "${CALICO_MODULE_NAME}" \ - --calico-kubernetes-module "${OCNE_CLUSTER_NAME}" \ - --calico-installation-config /vagrant/calico-config.yaml - - # Validate the Calico networking module - msg "Validating the Calico networking module: ${CALICO_MODULE_NAME}" - echo_do olcnectl module validate \ - --environment-name "${OCNE_ENV_NAME}" \ - --name "${CALICO_MODULE_NAME}" - - # Deploy the Calico networking module - msg "Deploying the Calico module: ${CALICO_MODULE_NAME} into ${OCNE_CLUSTER_NAME}" - echo_do olcnectl module install \ - --environment-name "${OCNE_ENV_NAME}" \ - --name "${CALICO_MODULE_NAME}" - fi - - # Multus networking module - if [[ ${DEPLOY_MULTUS} == 1 ]]; then - - if ! [ -f /vagrant/multus-config.yaml ]; then - echo_do "cat <<-EOF | tee /vagrant/multus-config.yaml - apiVersion: k8s.cni.cncf.io/v1 - kind: NetworkAttachmentDefinition - metadata: - name: bridge-conf - spec: - config: '{ - cniVersion: 0.3.1, - type: bridge, - bridge: mybr0, - ipam: { - type: host-local, - subnet: 192.168.12.0/24, - rangeStart: 192.168.12.10, - rangeEnd: 192.168.12.200 - } - }' -EOF" - fi - # Create the Multus networking module - msg "Creating the Multus networking module: ${MULTUS_MODULE_NAME}" - echo_do olcnectl module create \ - --environment-name "${OCNE_ENV_NAME}" \ - --module multus \ - --name "${MULTUS_MODULE_NAME}" \ - --multus-kubernetes-module "${OCNE_CLUSTER_NAME}" \ - --multus-installation-config /vagrant/multus-config.yaml - - # Validate the Multus networking module - msg "Validating the Multus networking module: ${MULTUS_MODULE_NAME}" - echo_do olcnectl module validate \ - --environment-name "${OCNE_ENV_NAME}" \ - --name "${MULTUS_MODULE_NAME}" - - # Deploy the Multus networking module - msg "Deploying the Multus module: ${MULTUS_MODULE_NAME} into ${OCNE_CLUSTER_NAME}" - echo_do olcnectl module install \ - --environment-name "${OCNE_ENV_NAME}" \ - --name "${MULTUS_MODULE_NAME}" - fi - - # Helm module (deprecated) - if [[ ${DEPLOY_HELM} == 1 ]]; then - - # Create the Helm module - msg "Creating the Helm module (deprecated): ${HELM_MODULE_NAME}" - echo_do olcnectl module create \ - --environment-name "${OCNE_ENV_NAME}" \ - --module helm \ - --name "${HELM_MODULE_NAME}" \ - --helm-kubernetes-module "${OCNE_CLUSTER_NAME}" - - # Validate the Helm module - msg "Validating the Helm module: ${HELM_MODULE_NAME}" - echo_do olcnectl module validate \ - --environment-name "${OCNE_ENV_NAME}" \ - --name "${HELM_MODULE_NAME}" - - # Deploy the Helm module - msg "Deploying the Helm module: ${HELM_MODULE_NAME} into ${OCNE_CLUSTER_NAME}" - echo_do olcnectl module install \ - --environment-name "${OCNE_ENV_NAME}" \ - --name "${HELM_MODULE_NAME}" - fi - - # Istio module - if [[ ${DEPLOY_ISTIO} == 1 ]]; then - - # Create the Istio module - msg "Creating the Istio module: ${ISTIO_MODULE_NAME}" - echo_do olcnectl module create \ - --environment-name "${OCNE_ENV_NAME}" \ - --module istio \ - --name "${ISTIO_MODULE_NAME}" \ - --istio-container-registry "${REGISTRY_OCNE}" \ - --istio-helm-module "${HELM_MODULE_NAME}" - - - # Validate the Istio module - msg "Validating the Istio module: ${ISTIO_MODULE_NAME}" - echo_do olcnectl module validate \ - --environment-name "${OCNE_ENV_NAME}" \ - --name "${ISTIO_MODULE_NAME}" - - # Deploy the Istio module - msg "Deploying the Istio module: ${ISTIO_MODULE_NAME} into ${OCNE_CLUSTER_NAME}" - echo_do olcnectl module install \ - --environment-name "${OCNE_ENV_NAME}" \ - --name "${ISTIO_MODULE_NAME}" - fi - - # MetalLB module - if [[ ${DEPLOY_METALLB} == 1 ]]; then - # Create MetalLB Configuration File - # https://metallb.universe.tf/configuration/ - echo_do "cat <<-EOF | tee /vagrant/metallb-config.yaml - address-pools: - - name: default - protocol: layer2 - addresses: - - ${SUBNET}.240-${SUBNET}.250 -EOF" - - # Create the MetalLB module - msg "Creating the MetalLB module: ${METALLB_MODULE_NAME}" - echo_do olcnectl module create \ - --environment-name "${OCNE_ENV_NAME}" \ - --module metallb \ - --name "${METALLB_MODULE_NAME}" \ - --metallb-kubernetes-module "${OCNE_CLUSTER_NAME}" \ - --metallb-config /vagrant/metallb-config.yaml - - msg "Removing MetalLB temporary configuration file" - echo_do rm -f /vagrant/metallb-config.yaml - - # Validate the MetalLB module - msg "Validating the MetalLB module: ${METALLB_MODULE_NAME}" - echo_do olcnectl module validate \ - --environment-name "${OCNE_ENV_NAME}" \ - --name "${METALLB_MODULE_NAME}" - - # Deploy the MetalLB module - msg "Deploying the MetalLB module: ${METALLB_MODULE_NAME} into ${OCNE_CLUSTER_NAME}" - echo_do olcnectl module install \ - --environment-name "${OCNE_ENV_NAME}" \ - --name "${METALLB_MODULE_NAME}" - fi - - # Gluster module (using Heketi) - if [[ ${DEPLOY_GLUSTER} == 1 ]]; then - - # Create the Gluster module - # using defaults url/user/secret-key: olcnectl module create --module gluster --help - msg "Creating the Gluster module (deprecated): ${GLUSTER_MODULE_NAME}" - HEKETI_CLI_SERVER="http://127.0.0.1:8080" - if [[ ${CONTROL_PLANE} == 0 ]]; then - # Standalone operator - HEKETI_CLI_SERVER="http://${SUBNET}.100:8080" - fi - echo_do olcnectl module create \ - --environment-name "${OCNE_ENV_NAME}" \ - --module gluster \ - --name "${GLUSTER_MODULE_NAME}" \ - --gluster-helm-module "${HELM_MODULE_NAME}" \ - --gluster-server-url "${HEKETI_CLI_SERVER}" - - # Validate the Gluster module - msg "Validating the Gluster module: ${GLUSTER_MODULE_NAME}" - echo_do olcnectl module validate \ - --environment-name "${OCNE_ENV_NAME}" \ - --name "${GLUSTER_MODULE_NAME}" - - # Deploy the Gluster module - msg "Deploying the Gluster module: ${GLUSTER_MODULE_NAME} into ${OCNE_CLUSTER_NAME}" - echo_do olcnectl module install \ - --environment-name "${OCNE_ENV_NAME}" \ - --name "${GLUSTER_MODULE_NAME}" - fi - -} - -####################################### -# Run Kubernetes fixups -# Globals: -# CONTROL_PLANES -# Arguments: -# None -# Returns: -# None -####################################### -fixups() { - local node - - msg "Copying admin.conf for vagrant user on control plane node(s)" - for node in ${CONTROL_PLANES//,/ }; do - echo_do ssh "${node}" "\"\ - mkdir -p ~/.kube; \ - sudo cp /etc/kubernetes/admin.conf ~/.kube/config; \ - sudo chown $(id -u):$(id -g) ~/.kube/config; \ - echo 'source <(kubectl completion bash)' >> ~/.bashrc; \ - echo 'alias k=kubectl' >> ~/.bashrc; \ - echo 'complete -F __start_kubectl k' >> ~/.bashrc; \ - echo 'command -v helm >/dev/null 2>&1 && source <(helm completion bash)' >> ~/.bashrc; \ - echo 'command -v istioctl >/dev/null 2>&1 && source <(istioctl completion bash)' >> ~/.bashrc; \ - \"" - done - - # Fix: /usr/libexec/crio/conmon doesn't exist - # conmon in @ol8_x86_64_appstream overrides @ol8_x86_64_olcne15 - msg "Change conmon from /usr/libexec/crio/conmon to /usr/bin/conmon in /etc/crio/crio.conf" - for node in ${CONTROL_PLANES//,/ } ${WORKERS//,/ }; do - echo_do ssh "${node}" "\"\ - sudo sed 's|/usr/libexec/crio/conmon|/usr/bin/conmon|' -i /etc/crio/crio.conf \ - && sudo systemctl restart crio.service \ - \"" - done - - msg "Starting kubectl proxy service on control plane nodes" - for node in ${CONTROL_PLANES//,/ }; do - # Expose the kubectl proxy to the host - echo_do ssh "${node}" "\"\ - sudo sed -i.bak 's/KUBECTL_PROXY_ARGS=--port 8001/KUBECTL_PROXY_ARGS=--port 8001 --accept-hosts=.* --address=0.0.0.0/' \ - /etc/systemd/system/kubectl-proxy.service.d/10-kubectl-proxy.conf \ - && sudo systemctl daemon-reload \ - && sudo systemctl enable --now kubectl-proxy.service \ - \"" - done - - # Fix: kubelet: "Unable to read config path" err="path does not exist, ignoring" path="/etc/kubernetes/manifests" - msg "Creating empty /etc/kubernetes/manifests directory on worker nodes" - for node in ${WORKERS//,/ }; do - echo_do ssh "${node}" "\"\ - [ -d /etc/kubernetes/manifests ] || sudo mkdir /etc/kubernetes/manifests - \"" - done - - # Fix: kubelet: summary_sys_containers.go: "Failed to get system container stats" - # err='failed to get cgroup stats for "/system.slice/kubelet.service": - # failed to get container info for "/system.slice/kubelet.service": - # unknown container "/system.slice/kubelet.service"' - # containerName="/system.slice/kubelet.service" - msg "Creating /etc/systemd/system/kubelet.service.d/11-cgroups.conf on K8s nodes" - for node in ${CONTROL_PLANES//,/ } ${WORKERS//,/ }; do - echo_do ssh "${node}" "\"\ - { cat <<-EOF | sudo tee /etc/systemd/system/kubelet.service.d/11-cgroups.conf - [Service] - CPUAccounting=true - MemoryAccounting=true - EOF - } \ - && sudo systemctl daemon-reload \ - && sudo systemctl restart kubelet \ - \"" - done - - # Fix: audit: type=1400 avc: denied { ioctl } for comm="iptables" path="/sys/fs/cgroup" dev="tmpfs" - msg "Fix AVC Denial on iptables" - for node in ${CONTROL_PLANES//,/ } ${WORKERS//,/ }; do - echo_do ssh "${node}" "\"\ - echo '(allow iptables_t cgroup_t (dir (ioctl)))' > /tmp/local_iptables.cil \ - && sudo semodule -i /tmp/local_iptables.cil \ - && rm -f /tmp/local_iptables.cil - \"" - done - - # Fix: Keepalived_vrrp: (VI_1) WARNING - equal priority advert received from remote host with our IP address. - if [[ ${MULTI_CONTROL_PLANE} == 1 ]]; then - msg "Fix Keepalived: remove unicast_src_ip from unicast_peers" - for node in ${CONTROL_PLANES//,/ }; do - echo_do ssh "${node}" "\"\ - sudo perl -i -ne 'print unless /^\s*$node\s*$/' /etc/keepalived/keepalived.conf \ - && sudo systemctl restart keepalived.service - \"" - done - fi - - # Fix: heketi: systemd[1]: /usr/lib/systemd/system/glusterd.service:21: Unknown lvalue 'StartLimitIntervalSec' in section 'Service' - if [[ ${DEPLOY_GLUSTER} == 1 ]]; then - msg "Removing StartLimitIntervalSec from /usr/lib/systemd/system/glusterd.service on Gluster nodes" - for node in ${WORKERS//,/ }; do - echo_do ssh "${node}" "\"\ - sudo sed -i '/^StartLimitIntervalSec=/d' /usr/lib/systemd/system/glusterd.service \ - && sudo systemctl daemon-reload \ - && sudo systemctl restart glusterd.service \ - \"" - done - - # Check if number of Gluster servers (Worker nodes) is less than 3, and patch K8s StorageClass. Default is 3 replicas. - NB_WORKERS=$(echo ${WORKERS} | awk -F',' '{print NF}') - if [[ ${NB_WORKERS} -lt "3" ]]; then - # https://kubernetes.io/docs/concepts/storage/storage-classes/#glusterfs - # https://github.com/kubernetes/examples/blob/master/staging/persistent-volume-provisioning/README.md - volumetype="none" # Distribute volume - if [[ ${NB_WORKERS} == "2" ]]; then - volumetype="replicate:2" # 2 replicas - fi - msg "Patching the Kubernetes hyperconverged storageclass volumetype to $volumetype" - node=${CONTROL_PLANES//,*/} - # K8s Storage Classes are immutable. Cannot: kubectl patch storageclasses hyperconverged -p '{"Parameters":{"volumetype":"replicate:2"}}' - echo_do ssh "${node}" "\"\ - kubectl get storageclasses hyperconverged -o=yaml | yq w - parameters.volumetype $volumetype > /vagrant/hyperconverged.yaml \ - && kubectl replace -f /vagrant/hyperconverged.yaml --force \ - && rm -f /vagrant/hyperconverged.yaml \ - \"" - fi - fi - - nodes="${CONTROL_PLANES},${WORKERS}" - if [[ ${CONTROL_PLANE} == 0 ]]; then - nodes="${SUBNET}.100,${nodes}" - fi - - # Fix: systemd: Started Session XX of user vagrant / session-XX.scope: - # systemd-logind: New session XX of user vagrant / Session XX logged out / Removed session XX - # https://access.redhat.com/solutions/1564823 - msg "Create discard filter to suppress user / session log entries in /var/log/messages" - echo 'if $programname == "systemd" and ($msg contains "Started Session" or $msg contains "scope: Succeeded") then stop' > /vagrant/ignore-systemd-session-slice.conf - echo 'if $programname == "systemd-logind" and ($msg contains "New session" or $msg contains "logged out. Waiting for processes to exit" or $msg contains "Removed session") then stop' > /vagrant/ignore-systemd-logind-session.conf - for node in ${nodes//,/ }; do - echo_do ssh "${node}" "\"\ - sudo cp /vagrant/ignore-systemd-session-slice.conf /etc/rsyslog.d/ \ - && sudo cp /vagrant/ignore-systemd-logind-session.conf /etc/rsyslog.d/ \ - && sudo systemctl restart rsyslog \ - \"" - done - echo_do rm -f /vagrant/ignore-systemd-session-slice.conf - echo_do rm -f /vagrant/ignore-systemd-logind-session.conf - - # Fix: firewalld: WARNING: AllowZoneDrifting is enabled. This is considered an insecure configuration option. - for node in ${nodes//,/ }; do - echo_do ssh "${node}" "\"\ - sudo sed -i 's/AllowZoneDrifting=yes/AllowZoneDrifting=no/' /etc/firewalld/firewalld.conf \ - && (sudo systemctl reload firewalld.service; true) \ - \"" - done - -} - -####################################### -# Cluster ready! -# Globals: -# CONTROL_PLANES -# Arguments: -# None -# Returns: -# None -####################################### -ready() { - local node api_server - - if [[ ${CONTROL_PLANE} == 1 ]]; then - api_server=${CONTROL_PLANES//,*/} - else - api_server=$(ip -f inet addr show eth1| sed -En -e 's/.*inet ([0-9.]+).*/\1/p') - fi - - node=${CONTROL_PLANES//,*/} - - msg "OCNE Modules deployed in this environment." - olcnectl module instances --api-server "${api_server}:8091" --environment-name "${OCNE_ENV_NAME}" - - msg "OCNE Pods deployed in this environment." - ssh vagrant@"${node}" kubectl get pods -A - - msg "Your Oracle Cloud Native Environment is operational." - ssh vagrant@"${node}" kubectl get nodes -o=wide -} - -####################################### -# Main -####################################### -main () { - parse_args "$@" - clean_networking - setup_repos - prerequisites - passwordless_ssh - if [[ ${OPERATOR} == 1 ]]; then - msg "Oracle Linux base pre-requisites complete,start provisioning nodes" - quick_install_ocne - deploy_modules - fixups - ready - fi -} - -main "$@"