From 80cf718e9aa7bd6f36f5804b316944ffa01bc584 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Harald=20Jens=C3=A5s?= Date: Thu, 30 Oct 2025 03:24:15 +0100 Subject: [PATCH] WIP - Add microshift installer --- 03-install_microshift.yml | 46 ++ bootstrap_microshift.yml | 30 + roles/microshift_installer/README.md | 167 +++++ roles/microshift_installer/defaults/main.yml | 53 ++ roles/microshift_installer/meta/main.yml | 33 + .../tasks/install_microshift.yml | 106 +++ roles/microshift_installer/tasks/main.yml | 26 + .../tasks/setup_controller.yml | 73 ++ scenarios/microshift-2-bm/README.md | 172 +++++ scenarios/microshift-2-bm/automation-vars.yml | 120 +++ scenarios/microshift-2-bm/bootstrap_vars.yml | 58 ++ scenarios/microshift-2-bm/heat_template.yaml | 701 ++++++++++++++++++ .../control-plane/control-plane.yaml | 471 ++++++++++++ .../control-plane/networking/metallb.yaml | 88 +++ .../control-plane/networking/nad.yaml | 134 ++++ .../control-plane/networking/nncp.yaml | 130 ++++ .../microshift-2-bm/test-operator/README.md | 42 ++ .../test-operator/automation-vars.yml | 380 ++++++++++ .../test-operator/manifests/nad.yaml | 21 + .../test-operator/tempest-tests.yml | 92 +++ 20 files changed, 2943 insertions(+) create mode 100644 03-install_microshift.yml create mode 100644 bootstrap_microshift.yml create mode 100644 roles/microshift_installer/README.md create mode 100644 roles/microshift_installer/defaults/main.yml create mode 100644 roles/microshift_installer/meta/main.yml create mode 100644 roles/microshift_installer/tasks/install_microshift.yml create mode 100644 roles/microshift_installer/tasks/main.yml create mode 100644 roles/microshift_installer/tasks/setup_controller.yml create mode 100644 scenarios/microshift-2-bm/README.md create mode 100644 scenarios/microshift-2-bm/automation-vars.yml create mode 100644 scenarios/microshift-2-bm/bootstrap_vars.yml create mode 100644 scenarios/microshift-2-bm/heat_template.yaml create mode 100644 scenarios/microshift-2-bm/manifests/control-plane/control-plane.yaml create mode 100644 scenarios/microshift-2-bm/manifests/control-plane/networking/metallb.yaml create mode 100644 scenarios/microshift-2-bm/manifests/control-plane/networking/nad.yaml create mode 100644 scenarios/microshift-2-bm/manifests/control-plane/networking/nncp.yaml create mode 100644 scenarios/microshift-2-bm/test-operator/README.md create mode 100644 scenarios/microshift-2-bm/test-operator/automation-vars.yml create mode 100644 scenarios/microshift-2-bm/test-operator/manifests/nad.yaml create mode 100644 scenarios/microshift-2-bm/test-operator/tempest-tests.yml diff --git a/03-install_microshift.yml b/03-install_microshift.yml new file mode 100644 index 00000000..27378dad --- /dev/null +++ b/03-install_microshift.yml @@ -0,0 +1,46 @@ +--- +# Copyright Red Hat, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +- name: Load stack outputs and add hosts to inventory + hosts: localhost + gather_facts: false + tasks: + - name: Load stack output vars from file + ansible.builtin.include_vars: + file: "{{ hotstack_work_dir | default(playbook_dir) }}/{{ stack_name }}-outputs.yaml" + name: stack_outputs + + - name: Add controller-0 to the Ansible inventory + ansible.builtin.add_host: "{{ stack_outputs.controller_ansible_host }}" + + - name: Add microshift-0 to the Ansible inventory + ansible.builtin.add_host: "{{ stack_outputs.microshift_ansible_host }}" + + - name: Store stack outputs for next play + ansible.builtin.set_fact: + stack_outputs: "{{ stack_outputs }}" + cacheable: true + +- name: Install MicroShift + hosts: microshifts,controllers + gather_facts: true + strategy: linear + any_errors_fatal: true + roles: + - role: microshift_installer + vars: + microshift_config: "{{ hostvars['localhost']['stack_outputs']['microshift_config'] }}" + microshift_nmstate_config: "{{ hostvars['localhost']['stack_outputs']['microshift_nmstate_config'] }}" diff --git a/bootstrap_microshift.yml b/bootstrap_microshift.yml new file mode 100644 index 00000000..5c8890d0 --- /dev/null +++ b/bootstrap_microshift.yml @@ -0,0 +1,30 @@ +--- +# Copyright Red Hat, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +- name: Bootstrap virtual infrastructure on Openstack cloud + ansible.builtin.import_playbook: 01-infra.yml + +- name: Bootstrap controller node + ansible.builtin.import_playbook: 02-bootstrap_controller.yml + +- name: Install MicroShift + ansible.builtin.import_playbook: 03-install_microshift.yml + +# - name: Deploy RedFish Virtual BMC +# ansible.builtin.import_playbook: 04-redfish_virtual_bmc.yml + +# - name: Deploy RHOSO +# ansible.builtin.import_playbook: 05_deploy_rhoso.yml diff --git a/roles/microshift_installer/README.md b/roles/microshift_installer/README.md new file mode 100644 index 00000000..80040c1b --- /dev/null +++ b/roles/microshift_installer/README.md @@ -0,0 +1,167 @@ +# MicroShift Installer Role + +This Ansible role installs and configures MicroShift on RHEL 9 systems. +It handles subscription management, package installation, network +configuration using nmstate, and initial cluster bootstrapping. + +## Requirements + +- RHEL 9.x system +- Valid Red Hat subscription with access to MicroShift repositories +- Network connectivity to Red Hat repositories +- Sufficient system resources (4GB RAM, 2 CPUs minimum recommended) + +## Role Variables + +### Required Variables + +```yaml +# Subscription Manager credentials +subscription_manager_org_id: "your-org-id" +subscription_manager_activation_key: "your-activation-key" + +# MicroShift configuration from Heat stack output (raw YAML for /etc/microshift/config.yaml) +microshift_config: + dns: + baseDomain: openstack.lab + node: + hostnameOverride: microshift-0 + nodeIP: 192.168.32.10 + +# nmstate network configuration from Heat stack output +microshift_nmstate_config: + interfaces: [] # nmstate interface configuration +``` + +### Optional Variables + +```yaml +# MicroShift version (default: "4.18") +microshift_installer_version: "4.18" + +# Service configuration +microshift_installer_service_enable: true +microshift_installer_service_state: started + +# Kubeconfig paths +microshift_installer_kubeconfig_path: "/var/lib/microshift/resources/kubeadmin/kubeconfig" + +# Cluster ready timeout (seconds) +microshift_installer_wait_timeout: 600 + +# Packages to install on MicroShift node +microshift_installer_packages: + - microshift + - microshift-networking + - microshift-selinux + - nmstate + - containernetworking-plugins +``` + +## Dependencies + +This role has no external dependencies beyond the built-in Ansible modules. + +## Example Playbook + +```yaml +- hosts: microshift_hosts + vars: + subscription_manager_org_id: "{{ lookup('env', 'RH_ORG_ID') }}" + subscription_manager_activation_key: "{{ lookup('env', 'RH_ACTIVATION_KEY') }}" + microshift_config: "{{ heat_stack_outputs.microshift_config }}" + microshift_nmstate_config: "{{ heat_stack_outputs.microshift_nmstate_config }}" + roles: + - microshift_installer +``` + +**Note**: The role handles privilege escalation internally using +`become: true` for tasks that require root access, so it's not +necessary to set `become: true` at the play level. + +## Installation Process + +The role performs the following steps on the MicroShift node: + +1. **System Registration**: Registers the RHEL system with Red Hat + Subscription Manager using the provided organization ID and activation + key. + +2. **Repository Enablement**: Enables the required repositories: + - `rhocp--for-rhel-9-x86_64-rpms` + - `fast-datapath-for-rhel-9-x86_64-rpms` + +3. **Package Installation**: Installs MicroShift and dependencies: + - microshift + - microshift-networking + - microshift-selinux + - nmstate + - containernetworking-plugins + +4. **Network Configuration**: Applies nmstate network configuration for: + - Physical interfaces + - VLAN interfaces + - Linux bridges + - IP addressing + +5. **MicroShift Configuration**: Creates `/etc/microshift/config.yaml` with: + - Base domain + - Hostname override + - Node IP address + +6. **Service Management**: Starts and enables the MicroShift service. + +7. **Cluster Bootstrap**: Waits for the cluster to be ready and the + kubeconfig to be available. + +The role also performs controller-specific tasks: + +1. **OC Client Installation**: Downloads and installs the `oc` CLI on the + controller node. + +2. **Kubeconfig Setup**: Fetches the kubeconfig from the MicroShift node + and sets it up on the controller at `~/.kube/config`. + +## Network Configuration + +The role uses nmstate to configure complex network topologies. The +`microshift_nmstate_config` should contain nmstate-compatible +configuration with interface definitions. This is typically generated from +the Heat stack output. + +Example network configuration includes: + +- Physical Ethernet interfaces +- VLAN-tagged interfaces for OpenStack networks +- Linux bridges for OVN and Ironic +- Static IP addressing + +## MicroShift Cluster Access + +After successful installation, the kubeconfig is available at: + +- System location: `/var/lib/microshift/resources/kubeadmin/kubeconfig` +- User location: `~/.kube/config` (copied by the role) + +Access the cluster: + +```bash +export KUBECONFIG=~/.kube/config +oc get nodes +oc get pods -A +``` + +## Integration with Hotstack + +This role is designed to be used with the Hotstack automation framework +as a replacement for `ocp_agent_installer` in MicroShift-based scenarios. +It expects the `microshift_config` variable to be populated from Heat +stack outputs. + +## Troubleshooting + +### Service Logs + +```bash +journalctl -u microshift -f +``` diff --git a/roles/microshift_installer/defaults/main.yml b/roles/microshift_installer/defaults/main.yml new file mode 100644 index 00000000..a0a27bc1 --- /dev/null +++ b/roles/microshift_installer/defaults/main.yml @@ -0,0 +1,53 @@ +--- +# Copyright Red Hat, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# Subscription Manager configuration +subscription_manager_org_id: "" +subscription_manager_activation_key: "" + +# MicroShift configuration from heat stack output +microshift_config: {} + +# MicroShift service configuration +microshift_installer_service_enable: true +microshift_installer_service_state: started + +# Kubeconfig location on MicroShift node +microshift_installer_kubeconfig_path: "/var/lib/microshift/resources/kubeadmin/kubeconfig" + +# Wait for cluster ready timeout (seconds) +microshift_installer_wait_timeout: 600 + +# Packages to install on MicroShift node +microshift_installer_packages: + - microshift + - microshift-networking + - microshift-selinux + - nmstate + - containernetworking-plugins + +# MicroShift version for repository enablement and client download +microshift_installer_version: "4.18" + +# OpenShift client download configuration (for controller) +microshift_installer_mirror_url: https://mirror.openshift.com/pub/openshift-v4/x86_64/clients/ocp +microshift_installer_client_url: "{{ microshift_installer_mirror_url }}/stable-{{ microshift_installer_version }}/openshift-client-linux.tar.gz" + +# Controller paths +microshift_installer_base_dir: /home/zuul +microshift_installer_bin_dir: "{{ microshift_installer_base_dir }}/bin" +microshift_installer_kube_config_dir: "{{ microshift_installer_base_dir }}/.kube" +microshift_installer_download_dir: "{{ microshift_installer_base_dir }}/downloads" diff --git a/roles/microshift_installer/meta/main.yml b/roles/microshift_installer/meta/main.yml new file mode 100644 index 00000000..6f772763 --- /dev/null +++ b/roles/microshift_installer/meta/main.yml @@ -0,0 +1,33 @@ +--- +# Copyright Red Hat, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +galaxy_info: + author: OpenStack K8s Operators + description: Install and configure MicroShift on RHEL 9 + company: Red Hat + license: Apache-2.0 + min_ansible_version: "2.9" + platforms: + - name: EL + versions: + - "9" + galaxy_tags: + - openshift + - microshift + - kubernetes + - edge + +dependencies: [] diff --git a/roles/microshift_installer/tasks/install_microshift.yml b/roles/microshift_installer/tasks/install_microshift.yml new file mode 100644 index 00000000..861348a5 --- /dev/null +++ b/roles/microshift_installer/tasks/install_microshift.yml @@ -0,0 +1,106 @@ +--- +# Copyright Red Hat, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# This file contains tasks that run on the MicroShift node + +- name: Assert required variables are defined + ansible.builtin.assert: + that: + - subscription_manager_org_id is defined + - subscription_manager_org_id | length > 0 + - subscription_manager_activation_key is defined + - subscription_manager_activation_key | length > 0 + - microshift_config is defined + - microshift_config is mapping + - microshift_config | length > 0 + - microshift_nmstate_config is defined + - microshift_nmstate_config is mapping + - microshift_nmstate_config | length > 0 + fail_msg: >- + Required variables not defined or invalid. + Check subscription_manager_org_id, subscription_manager_activation_key, + microshift_config, and microshift_nmstate_config. + +- name: Register system with subscription-manager + ansible.builtin.command: + cmd: >- + subscription-manager register + --org="{{ subscription_manager_org_id }}" + --activationkey="{{ subscription_manager_activation_key }}" + +- name: Enable required repositories + loop: + - "rhocp-{{ microshift_installer_version }}-for-rhel-9-x86_64-rpms" + - "fast-datapath-for-rhel-9-x86_64-rpms" + ansible.builtin.command: + cmd: subscription-manager repos --enable="{{ item }}" + +- name: Install MicroShift and dependencies + ansible.builtin.dnf: + name: "{{ microshift_installer_packages }}" + state: present + +- name: Create hotstack configuration directory + ansible.builtin.file: + path: /etc/hotstack + state: directory + mode: '0755' + owner: root + group: root + +- name: Write nmstate network configuration + ansible.builtin.copy: + content: "{{ microshift_nmstate_config | to_nice_yaml }}" + dest: /etc/hotstack/nmstate-config.yaml + owner: root + group: root + mode: '0644' + +- name: Apply nmstate network configuration + ansible.builtin.command: + cmd: nmstatectl apply /etc/hotstack/nmstate-config.yaml + +- name: Create MicroShift configuration directory + ansible.builtin.file: + path: /etc/microshift + state: directory + mode: '0755' + owner: root + group: root + +- name: Configure MicroShift + ansible.builtin.copy: + content: "{{ microshift_config | to_nice_yaml }}" + dest: /etc/microshift/config.yaml + mode: '0644' + owner: root + group: root + +- name: Start and enable MicroShift service + ansible.builtin.systemd: + name: microshift + state: "{{ microshift_installer_service_state }}" + enabled: "{{ microshift_installer_service_enable }}" + +- name: Wait for MicroShift to be ready + ansible.builtin.wait_for: + path: "{{ microshift_installer_kubeconfig_path }}" + timeout: "{{ microshift_installer_wait_timeout }}" + +- name: Read kubeconfig content + register: kubeconfig_content + ansible.builtin.slurp: + src: "{{ microshift_installer_kubeconfig_path }}" diff --git a/roles/microshift_installer/tasks/main.yml b/roles/microshift_installer/tasks/main.yml new file mode 100644 index 00000000..a1b58de5 --- /dev/null +++ b/roles/microshift_installer/tasks/main.yml @@ -0,0 +1,26 @@ +--- +# Copyright Red Hat, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +- name: Install and configure MicroShift on target node + when: "'microshifts' in group_names" + become: true + block: + - name: Include MicroShift installation tasks + ansible.builtin.include_tasks: install_microshift.yml + +- name: Set up oc client and kubeconfig on controller + when: "'controllers' in group_names" + ansible.builtin.include_tasks: setup_controller.yml diff --git a/roles/microshift_installer/tasks/setup_controller.yml b/roles/microshift_installer/tasks/setup_controller.yml new file mode 100644 index 00000000..a65d0321 --- /dev/null +++ b/roles/microshift_installer/tasks/setup_controller.yml @@ -0,0 +1,73 @@ +--- +# Copyright Red Hat, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# This file contains tasks that run on the controller + +- name: Create directories on controller + ansible.builtin.file: + path: "{{ item }}" + state: directory + mode: '0755' + loop: + - "{{ microshift_installer_bin_dir }}" + - "{{ microshift_installer_kube_config_dir }}" + - "{{ microshift_installer_download_dir }}" + +- name: Download OpenShift client + ansible.builtin.get_url: + url: "{{ microshift_installer_client_url }}" + dest: "{{ microshift_installer_download_dir }}/openshift-client-linux.tar.gz" + mode: '0644' + +- name: Extract oc client to bin directory + ansible.builtin.unarchive: + src: "{{ microshift_installer_download_dir }}/openshift-client-linux.tar.gz" + dest: "{{ microshift_installer_bin_dir }}" + remote_src: true + creates: "{{ microshift_installer_bin_dir }}/oc" + +- name: Write kubeconfig on controller + ansible.builtin.copy: + content: "{{ hostvars[groups['microshifts'][0]]['kubeconfig_content']['content'] | b64decode }}" + dest: "{{ microshift_installer_kube_config_dir }}/config" + mode: '0600' + +- name: Update kubeconfig server address + vars: + microshift_api_address: "{{ hostvars[groups['microshifts'][0]]['ansible_host'] }}" + ansible.builtin.replace: + path: "{{ microshift_installer_kube_config_dir }}/config" + regexp: 'https://127\.0\.0\.1:6443' + replace: "https://{{ microshift_api_address }}:6443" + +- name: Configure bash completion for oc + become: true + ansible.builtin.shell: | + {{ microshift_installer_bin_dir }}/oc completion bash > /etc/bash_completion.d/oc_bash_completion + args: + creates: /etc/bash_completion.d/oc_bash_completion + +- name: Verify cluster access from controller + changed_when: false + register: nodes_output + environment: + KUBECONFIG: "{{ microshift_installer_kube_config_dir }}/config" + ansible.builtin.command: + cmd: "{{ microshift_installer_bin_dir }}/oc get nodes" + +- name: Display cluster nodes + ansible.builtin.debug: + var: nodes_output.stdout_lines diff --git a/scenarios/microshift-2-bm/README.md b/scenarios/microshift-2-bm/README.md new file mode 100644 index 00000000..ba85bb15 --- /dev/null +++ b/scenarios/microshift-2-bm/README.md @@ -0,0 +1,172 @@ +# microshift-2-bm Scenario + +## Overview + +A MicroShift scenario designed to test OpenStack Ironic bare metal provisioning +with 2 dedicated Ironic nodes. This scenario validates the complete OpenStack +bare metal lifecycle including node enrollment, provisioning, and comprehensive +Tempest testing using MicroShift instead of a full OpenShift cluster. + +## Architecture + + +```mermaid +graph TD + Internet[("Internet")] + Router{{"Neutron
Router"}} + + MachineNet["Machine Network
192.168.32.0/24"] + CtlPlane["CtlPlane Network
192.168.122.0/24"] + VLANNets["VLAN Trunk Networks
Internal API: 172.17.0.0/24
Storage: 172.18.0.0/24
Tenant: 172.19.0.0/24"] + IronicNet["Ironic Network
172.20.1.0/24"] + + Controller["Controller
192.168.32.254
DNS/HAProxy"] + MicroShift["MicroShift Host
192.168.32.10
RHEL 9.6"] + IronicNodes["Ironic Nodes x2
Virtual Baremetal"] + + LVM["TopoLVM
20GB"] + CinderVols["Cinder Volumes x3
20GB each"] + + Internet --- Router + + Router --- MachineNet + Router --- CtlPlane + Router --- VLANNets + Router --- IronicNet + + MachineNet --- Controller + MachineNet --- MicroShift + CtlPlane --- MicroShift + VLANNets --- MicroShift + IronicNet --- MicroShift + IronicNet --- IronicNodes + + MicroShift --- LVM + MicroShift --- CinderVols + + style Controller fill:#4A90E2,stroke:#2E5C8A,stroke-width:3px,color:#fff + style MicroShift fill:#F5A623,stroke:#C87D0E,stroke-width:3px,color:#fff + style IronicNodes fill:#9B59B6,stroke:#6C3A82,stroke-width:2px,color:#fff + style Router fill:#27AE60,stroke:#1E8449,stroke-width:3px,color:#fff +``` + + +### Component Details + +- **Controller**: Hotstack controller providing DNS, load balancing, and + orchestration services +- **MicroShift Host**: RHEL 9.6 host running MicroShift with the complete + OpenStack control plane +- **Ironic Nodes**: 2 virtual bare metal nodes for testing Ironic provisioning + workflows + +## Features + +- MicroShift lightweight Kubernetes distribution +- OpenStack Ironic bare metal provisioning service +- Virtual BMC using sushy-tools for RedFish emulation +- Comprehensive Tempest testing (scenario and API tests) +- Complete OpenStack service stack (Nova, Neutron, Glance, Swift, etc.) +- TopoLVM for local storage management +- Multi-network setup for OpenStack services +- Automatic node enrollment and lifecycle management + +## Networks + +- **machine-net**: 192.168.32.0/24 (MicroShift cluster network) +- **ctlplane-net**: 192.168.122.0/24 (OpenStack control plane) +- **internal-api-net**: 172.17.0.0/24 (OpenStack internal services) +- **storage-net**: 172.18.0.0/24 (Storage backend communication) +- **tenant-net**: 172.19.0.0/24 (Tenant network traffic) +- **ironic-net**: 172.20.1.0/24 (Bare metal provisioning network) + +## OpenStack Services + +This scenario deploys a comprehensive OpenStack environment: + +### Core Services + +- **Keystone**: Identity service with LoadBalancer on Internal API +- **Nova**: Compute service with Ironic driver for bare metal +- **Neutron**: Networking service with OVN backend +- **Glance**: Image service with Swift backend +- **Swift**: Object storage service +- **Placement**: Resource placement service + +### Bare Metal Services + +- **Ironic**: Bare metal provisioning service +- **Ironic Inspector**: Hardware inspection service +- **Ironic Neutron Agent**: Network management for bare metal + +## Ironic Testing + +### Node Configuration + +- **2 Ironic Nodes**: Virtual instances with sushy-tools RedFish BMC +- **Flavor**: hotstack.medium (configurable) +- **Network**: Connected to dedicated Ironic provisioning network + +### Test Scenarios + +The scenario includes Tempest testing: + +#### Scenario Tests + +- Baremetal basic operations testing +- Instance lifecycle management +- Network connectivity validation +- Power management testing + +#### API Tests + +- Ironic API functionality validation +- Node management operations +- Port and allocation management +- Hardware inspection workflows + +## Storage Configuration + +- **TopoLVM**: Local volume management for OpenStack services +- **Cinder Volumes**: Additional block storage on `/dev/vdc`, `/dev/vdd`, `/dev/vde` +- **Swift Storage**: Object storage for Glance images +- **Database Storage**: Persistent storage for Galera clusters + +## Differences from sno-2-bm + +This scenario differs from `sno-2-bm` in the following ways: + +1. **MicroShift instead of SNO**: Uses MicroShift (lightweight Kubernetes) + instead of Single Node OpenShift +2. **RHEL 9.6 Base**: Uses `rhel-9.6-x86_64-kvm` image instead of iPXE boot +3. **No PXE Boot**: Does not require PXE boot DHCP options since MicroShift + boots directly from RHEL image +4. **Simplified Deployment**: Lighter weight deployment suitable for testing + and development + +## Usage + +```bash +# Deploy the scenario +ansible-playbook -i inventory.yml bootstrap.yml \ + -e @scenarios/microshift-2-bm/bootstrap_vars.yml \ + -e @~/cloud-secrets.yaml + +# Run comprehensive tests +ansible-playbook -i inventory.yml 06-test-operator.yml \ + -e @scenarios/microshift-2-bm/bootstrap_vars.yml \ + -e @~/cloud-secrets.yaml +``` + +## Configuration Files + +- `bootstrap_vars.yml`: Infrastructure and MicroShift configuration +- `automation-vars.yml`: Hotloop deployment stages +- `heat_template.yaml`: OpenStack infrastructure template +- `manifests/control-plane/control-plane.yaml`: OpenStack service configuration +- `test-operator/automation-vars.yml`: Comprehensive test automation +- `test-operator/tempest-tests.yml`: Tempest test specifications + +This scenario provides a complete environment for validating OpenStack bare +metal provisioning capabilities in a MicroShift deployment with comprehensive +testing automation. diff --git a/scenarios/microshift-2-bm/automation-vars.yml b/scenarios/microshift-2-bm/automation-vars.yml new file mode 100644 index 00000000..e3256e61 --- /dev/null +++ b/scenarios/microshift-2-bm/automation-vars.yml @@ -0,0 +1,120 @@ +--- +stages: + - name: TopoLVM Dependencies + stages: >- + {{ + lookup("ansible.builtin.template", + "common/stages/topolvm-deps-stages.yaml.j2") + }} + + - name: Dependencies + stages: >- + {{ + lookup("ansible.builtin.template", + "common/stages/deps-stages.yaml.j2") + }} + + - name: Cinder LVM + stages: >- + {{ + lookup("ansible.builtin.file", + "common/stages/cinder-lvm-label-stages.yaml") + }} + + - name: TopoLVM + stages: >- + {{ + lookup("ansible.builtin.template", + "common/stages/topolvm-stages.yaml.j2") + }} + + - name: OLM Openstack + stages: >- + {{ + lookup("ansible.builtin.template", + "common/stages/olm-openstack-stages.yaml.j2") + }} + + - name: NodeNetworkConfigurationPolicy (nncp) + documentation: | + Apply node network configuration policies to configure host networking. + Waits for all policies to be successfully configured. + manifest: manifests/control-plane/networking/nncp.yaml + wait_conditions: + - >- + oc wait -n openstack nncp -l osp/nncm-config-type=standard + --for jsonpath='{.status.conditions[0].reason}'=SuccessfullyConfigured + --timeout=180s + + - name: NetworkAttchmentDefinition (NAD) + documentation: | + Create network attachment definitions for OpenStack services. + Defines additional network interfaces for pods. + manifest: manifests/control-plane/networking/nad.yaml + + - name: MetalLB - L2Advertisement and IPAddressPool + documentation: | + Configure MetalLB load balancer with IP address pools and L2 advertisements. + Enables external access to OpenStack services. + manifest: manifests/control-plane/networking/metallb.yaml + + - name: OpenstackControlPlane + documentation: | + Deploy the OpenStack control plane with all core services. + Waits for the control plane to be fully ready before proceeding. + manifest: manifests/control-plane/control-plane.yaml + wait_conditions: + - >- + oc wait -n openstack openstackcontrolplane controlplane + --for condition=Ready --timeout=30m + + - name: Update openstack-operators OLM + stages: >- + {{ + lookup('ansible.builtin.template', + 'common/stages/openstack-olm-update.yaml.j2') + }} + run_conditions: + - >- + {{ + openstack_operators_update is defined and + openstack_operators_update | bool + }} + + - name: Wait for condition MinorUpdateAvailable True + documentation: | + Wait for OpenStack version to indicate a minor update is available. + Required before proceeding with version updates. + wait_conditions: + - >- + oc -n openstack wait openstackversions.core.openstack.org controlplane + --for=condition=MinorUpdateAvailable=True --timeout=10m + run_conditions: + - "{{ openstack_update is defined and openstack_update | bool }}" + + - name: "Minor update :: Create OpenStackVersion patch" + documentation: | + This creates a patch file `{{ manifests_dir }}/patches/openstack_version_patch.yaml` + If `openstack_update_custom_images` is defined it will populate the customContainerImages + in the OpenstackVersion YAML patch. + shell: >- + {{ + lookup('ansible.builtin.template', + 'common/scripts/create_openstack_version_patch.sh.j2') + }} + run_conditions: + - "{{ openstack_update is defined and openstack_update | bool }}" + + - name: "Minor update :: Update the target version in the OpenStackVersion custom resource (CR)" + documentation: | + The `hotstack-openstack-version-patch` script will get the `availableVersion` + and us it to replace the string `__TARGET_VERSION__` in the patch file and + apply the patch using `oc patch` command. + command: >- + hotstack-openstack-version-patch --namespace openstack --name controlplane + --file {{ manifests_dir }}/patches/openstack_version_patch.yaml + wait_conditions: + - oc -n openstack wait openstackversions.core.openstack.org controlplane + --for=condition=Ready --timeout=10m + run_conditions: + - "{{ openstack_update is defined and openstack_update | bool }}" diff --git a/scenarios/microshift-2-bm/bootstrap_vars.yml b/scenarios/microshift-2-bm/bootstrap_vars.yml new file mode 100644 index 00000000..06c9c680 --- /dev/null +++ b/scenarios/microshift-2-bm/bootstrap_vars.yml @@ -0,0 +1,58 @@ +--- +os_cloud: default +os_floating_network: public +os_router_external_network: public + +scenario: microshift-2-bm +scenario_dir: scenarios +stack_template_path: "{{ scenario_dir }}/{{ scenario }}/heat_template.yaml" +automation_vars_file: "{{ scenario_dir }}/{{ scenario }}/automation-vars.yml" +test_operator_automation_vars_file: "{{ scenario_dir }}/{{ scenario }}/test-operator/automation-vars.yml" + +# Deployment type: 'ocp' or 'microshift' +hotstack_openshift_deployment_type: microshift + +openstack_operators_image: quay.io/openstack-k8s-operators/openstack-operator-index:latest +openstack_operator_channel: alpha +openstack_operator_starting_csv: null + +# MicroShift version +microshift_installer_version: "4.18" + +ntp_servers: [] +dns_servers: + - 8.8.8.8 + - 8.8.4.4 + +pull_secret_file: ~/pull-secret.txt + +ovn_k8s_gateway_config_host_routing: true +enable_iscsi: true +enable_multipath: true + +cinder_volume_pvs: + - /dev/vdc + - /dev/vdd + - /dev/vde + +stack_name: "hs-{{ scenario }}-{{ zuul.build[:8] | default('no-zuul') }}" +stack_parameters: + # On misconfigured clouds, uncomment these to avoid issues. + # Ref: https://access.redhat.com/solutions/7059376 + # net_value_specs: + # mtu: 1442 + dns_servers: "{{ dns_servers }}" + ntp_servers: "{{ ntp_servers }}" + controller_ssh_pub_key: "{{ controller_ssh_pub_key | default('') }}" + router_external_network: "{{ os_router_external_network | default('public') }}" + floating_ip_network: "{{ os_floating_network | default('public') }}" + controller_params: + image: hotstack-controller + flavor: hotstack.small + microshift_params: + image: rhel-9.6-x86_64-kvm + flavor: hotstack.xxlarge + ironic_params: + image: CentOS-Stream-GenericCloud-9 + cd_image: sushy-tools-blank-image + flavor: hotstack.medium diff --git a/scenarios/microshift-2-bm/heat_template.yaml b/scenarios/microshift-2-bm/heat_template.yaml new file mode 100644 index 00000000..b2f15d82 --- /dev/null +++ b/scenarios/microshift-2-bm/heat_template.yaml @@ -0,0 +1,701 @@ +--- +heat_template_version: rocky + +description: > + Heat template to set up 3-node infrastructure, 1x Controller, 1x MicroShift, 1x Compute + +parameters: + dns_servers: + type: comma_delimited_list + default: + - 8.8.8.8 + - 8.8.4.4 + ntp_servers: + type: comma_delimited_list + default: [] + controller_ssh_pub_key: + type: string + dataplane_ssh_pub_key: + type: string + router_external_network: + type: string + default: public + floating_ip_network: + type: string + default: public + net_value_specs: + type: json + default: {} + + controller_params: + type: json + default: + image: hotstack-controller + flavor: hotstack.small + nat64_appliance_params: + type: json + default: + image: nat64-appliance + flavor: hotstack.small + ocp_master_params: + type: json + default: + image: ipxe-boot-usb + flavor: hotstack.xxlarge + microshift_params: + type: json + default: + image: rhel-9.6-x86_64-kvm + flavor: hotstack.xxlarge + ocp_worker_params: + type: json + default: + image: ipxe-boot-usb + flavor: hotstack.xxlarge + compute_params: + type: json + default: + image: CentOS-Stream-GenericCloud-9 + flavor: hotstack.large + networker_params: + type: json + default: + image: CentOS-Stream-GenericCloud-9 + flavor: hotstack.small + bmh_params: + type: json + default: + image: CentOS-Stream-GenericCloud-9 + cd_image: sushy-tools-blank-image + flavor: hotstack.medium + ironic_params: + type: json + default: + image: CentOS-Stream-GenericCloud-9 + cd_image: sushy-tools-blank-image + flavor: hotstack.medium + +resources: + # + # Networks + # + machine-net: + type: OS::Neutron::Net + properties: + port_security_enabled: false + value_specs: {get_param: net_value_specs} + + ctlplane-net: + type: OS::Neutron::Net + properties: + port_security_enabled: false + value_specs: {get_param: net_value_specs} + + internal-api-net: + type: OS::Neutron::Net + properties: + port_security_enabled: false + value_specs: {get_param: net_value_specs} + + storage-net: + type: OS::Neutron::Net + properties: + port_security_enabled: false + value_specs: {get_param: net_value_specs} + + tenant-net: + type: OS::Neutron::Net + properties: + port_security_enabled: false + value_specs: {get_param: net_value_specs} + + ironic-net: + type: OS::Neutron::Net + properties: + port_security_enabled: false + value_specs: {get_param: net_value_specs} + + # + # Subnets + # + machine-subnet: + type: OS::Neutron::Subnet + properties: + network: {get_resource: machine-net} + ip_version: 4 + cidr: 192.168.32.0/24 + enable_dhcp: true + dns_nameservers: + - 192.168.32.254 + + ctlplane-subnet: + type: OS::Neutron::Subnet + properties: + network: {get_resource: ctlplane-net} + ip_version: 4 + cidr: 192.168.122.0/24 + enable_dhcp: false + allocation_pools: + - start: 192.168.122.100 + end: 192.168.122.150 + dns_nameservers: + - 192.168.122.80 + + internal-api-subnet: + type: OS::Neutron::Subnet + properties: + network: {get_resource: internal-api-net} + ip_version: 4 + cidr: 172.17.0.0/24 + enable_dhcp: false + allocation_pools: + - start: 172.17.0.100 + end: 172.17.0.150 + + storage-subnet: + type: OS::Neutron::Subnet + properties: + network: {get_resource: storage-net} + ip_version: 4 + cidr: 172.18.0.0/24 + enable_dhcp: false + allocation_pools: + - start: 172.18.0.100 + end: 172.18.0.150 + + tenant-subnet: + type: OS::Neutron::Subnet + properties: + network: {get_resource: tenant-net} + ip_version: 4 + cidr: 172.19.0.0/24 + enable_dhcp: false + allocation_pools: + - start: 172.19.0.100 + end: 172.19.0.150 + + ironic-subnet: + type: OS::Neutron::Subnet + properties: + network: {get_resource: ironic-net} + ip_version: 4 + cidr: 172.20.1.0/24 + enable_dhcp: false + allocation_pools: [{start: 172.20.1.100, end: 172.20.1.150}] + + # + # Routers + # + router: + type: OS::Neutron::Router + properties: + admin_state_up: true + external_gateway_info: + network: {get_param: router_external_network} + # enable_snat: true + + machine-net-router-interface: + type: OS::Neutron::RouterInterface + properties: + router: {get_resource: router} + subnet: {get_resource: machine-subnet} + + ctlplane-net-router-interface: + type: OS::Neutron::RouterInterface + properties: + router: {get_resource: router} + subnet: {get_resource: ctlplane-subnet} + + ironic-net-router-interface: + type: OS::Neutron::RouterInterface + properties: + router: {get_resource: router} + subnet: {get_resource: ironic-subnet} + + # + # Instances + # + controller_users: + type: OS::Heat::CloudConfig + properties: + cloud_config: + users: + - default + - name: zuul + gecos: "Zuul user" + sudo: ALL=(ALL) NOPASSWD:ALL + ssh_authorized_keys: + - {get_param: controller_ssh_pub_key} + + controller-write-files: + type: OS::Heat::CloudConfig + properties: + cloud_config: + write_files: + - path: /etc/dnsmasq.conf + content: | + # dnsmasq service config + # Include all files in /etc/dnsmasq.d except RPM backup files + conf-dir=/etc/dnsmasq.d,.rpmnew,.rpmsave,.rpmorig + no-resolv + owner: root:dnsmasq + - path: /etc/dnsmasq.d/forwarders.conf + content: + str_replace: + template: | + # DNS forwarders records + server=$dns1 + server=$dns2 + params: + $dns1: {get_param: [dns_servers, 0]} + $dns2: {get_param: [dns_servers, 1]} + owner: root:dnsmasq + - path: /etc/dnsmasq.d/host_records.conf + content: + str_replace: + template: | + # Host records + host-record=controller-0.openstack.lab,$controller0 + host-record=api.microshift.openstack.lab,$microshift0 + host-record=api-int.microshift.openstack.lab,$microshift0 + host-record=microshift-0.microshift.openstack.lab,$microshift0 + params: + $controller0: {get_attr: [controller-machine-port, fixed_ips, 0, ip_address]} + $microshift0: {get_attr: [microshift0-machine-port, fixed_ips, 0, ip_address]} + owner: root:dnsmasq + - path: /etc/dnsmasq.d/wildcard_records.conf + content: + str_replace: + template: | + # Wildcard records + address=/apps.microshift.openstack.lab/$addr + params: + $addr: {get_attr: [microshift0-machine-port, fixed_ips, 0, ip_address]} + owner: root:dnsmasq + - path: /etc/resolv.conf + content: | + nameserver: 127.0.0.1 + owner: root:root + - path: /etc/NetworkManager/conf.d/98-rc-manager.conf + content: | + [main] + rc-manager=unmanaged + owner: root:root + + + controller-runcmd: + type: OS::Heat::CloudConfig + properties: + cloud_config: + runcmd: + - ['systemctl', 'enable', 'dnsmasq.service'] + - ['systemctl', 'start', 'dnsmasq.service'] + - ['setenforce', 'permissive'] + + controller-init: + type: OS::Heat::MultipartMime + properties: + parts: + - config: {get_resource: controller_users} + - config: {get_resource: controller-write-files} + - config: {get_resource: controller-runcmd} + + controller-machine-port: + type: OS::Neutron::Port + properties: + network: {get_resource: machine-net} + mac_address: "fa:16:9e:81:f6:05" + fixed_ips: + - ip_address: 192.168.32.254 + + controller-floating-ip: + depends_on: machine-net-router-interface + type: OS::Neutron::FloatingIP + properties: + floating_network: {get_param: floating_ip_network} + port_id: {get_resource: controller-machine-port} + + controller: + type: OS::Nova::Server + properties: + image: {get_param: [controller_params, image]} + flavor: {get_param: [controller_params, flavor]} + networks: + - port: {get_resource: controller-machine-port} + user_data_format: RAW + user_data: {get_resource: controller-init} + + # MicroShift Host + + microshift_users: + type: OS::Heat::CloudConfig + properties: + cloud_config: + users: + - default + - name: zuul + gecos: "Zuul user" + sudo: ALL=(ALL) NOPASSWD:ALL + ssh_authorized_keys: + - {get_param: controller_ssh_pub_key} + + microshift-hostname: + type: OS::Heat::CloudConfig + properties: + cloud_config: + hostname: microshift-0 + fqdn: microshift-0.openstack.lab + + microshift-init: + type: OS::Heat::MultipartMime + properties: + parts: + - config: {get_resource: microshift_users} + - config: {get_resource: microshift-hostname} + + microshift0-machine-port: + type: OS::Neutron::Port + properties: + network: {get_resource: machine-net} + port_security_enabled: false + mac_address: "fa:16:9e:81:f6:10" + fixed_ips: + - ip_address: 192.168.32.10 + + microshift0-ctlplane-trunk-parent-port: + type: OS::Neutron::Port + properties: + network: {get_resource: ctlplane-net} + port_security_enabled: false + fixed_ips: + - ip_address: 192.168.122.10 + + microshift0-internal-api-port: + type: OS::Neutron::Port + properties: + network: {get_resource: internal-api-net} + port_security_enabled: false + fixed_ips: + - ip_address: 172.17.0.10 + + microshift0-storage-port: + type: OS::Neutron::Port + properties: + network: {get_resource: storage-net} + port_security_enabled: false + fixed_ips: + - ip_address: 172.18.0.10 + + microshift0-tenant-port: + type: OS::Neutron::Port + properties: + network: {get_resource: tenant-net} + port_security_enabled: false + fixed_ips: + - ip_address: 172.19.0.10 + + microshift0-trunk0: + type: OS::Neutron::Trunk + properties: + port: {get_resource: microshift0-ctlplane-trunk-parent-port} + sub_ports: + - port: {get_resource: microshift0-internal-api-port} + segmentation_id: 20 + segmentation_type: vlan + - port: {get_resource: microshift0-storage-port} + segmentation_id: 21 + segmentation_type: vlan + - port: {get_resource: microshift0-tenant-port} + segmentation_id: 22 + segmentation_type: vlan + + microshift0-ironic-port: + type: OS::Neutron::Port + properties: + network: {get_resource: ironic-net} + port_security_enabled: false + fixed_ips: [{ip_address: 172.20.1.10}] + + microshift0-lvms-vol0: + type: OS::Cinder::Volume + properties: + size: 20 + + microshift0-cinder-vol0: + type: OS::Cinder::Volume + properties: + size: 20 + + microshift0-cinder-vol1: + type: OS::Cinder::Volume + properties: + size: 20 + + microshift0-cinder-vol2: + type: OS::Cinder::Volume + properties: + size: 20 + + microshift0: + type: OS::Nova::Server + properties: + image: {get_param: [microshift_params, image]} + flavor: {get_param: [microshift_params, flavor]} + block_device_mapping_v2: + - boot_index: -1 + device_type: disk + volume_id: {get_resource: microshift0-lvms-vol0} + - boot_index: -1 + device_type: disk + volume_id: {get_resource: microshift0-cinder-vol0} + - boot_index: -1 + device_type: disk + volume_id: {get_resource: microshift0-cinder-vol1} + - boot_index: -1 + device_type: disk + volume_id: {get_resource: microshift0-cinder-vol2} + networks: + - port: {get_resource: microshift0-machine-port} + - port: {get_attr: [microshift0-trunk0, port_id]} + - port: {get_resource: microshift0-ironic-port} + user_data_format: RAW + user_data: {get_resource: microshift-init} + + # + # Ironics + # + ironic0-port: + type: OS::Neutron::Port + properties: + network: {get_resource: ironic-net} + port_security_enabled: false + + ironic0: + type: OS::Nova::Server + properties: + flavor: {get_param: [ironic_params, flavor]} + block_device_mapping_v2: + - device_type: disk + boot_index: 1 + image_id: {get_param: [ironic_params, image]} + volume_size: 40 + delete_on_termination: true + - device_type: cdrom + disk_bus: scsi + boot_index: 0 + image_id: {get_param: [ironic_params, cd_image]} + volume_size: 5 + delete_on_termination: true + networks: + - port: {get_resource: ironic0-port} + + ironic1-port: + type: OS::Neutron::Port + properties: + network: {get_resource: ironic-net} + port_security_enabled: false + + ironic1: + type: OS::Nova::Server + properties: + flavor: {get_param: [ironic_params, flavor]} + block_device_mapping_v2: + - device_type: disk + boot_index: 1 + image_id: {get_param: [ironic_params, image]} + volume_size: 40 + delete_on_termination: true + - device_type: cdrom + disk_bus: scsi + boot_index: 0 + image_id: {get_param: [ironic_params, cd_image]} + volume_size: 5 + delete_on_termination: true + networks: + - port: {get_resource: ironic1-port} + + +outputs: + controller_floating_ip: + description: Controller Floating IP + value: {get_attr: [controller-floating-ip, floating_ip_address]} + + microshift_config: + description: MicroShift configuration (raw YAML for /etc/microshift/config.yaml) + value: + dns: + baseDomain: microshift.openstack.lab + node: + hostnameOverride: microshift-0 + nodeIP: {get_attr: [microshift0-machine-port, fixed_ips, 0, ip_address]} + + microshift_nmstate_config: + description: nmstate network configuration for MicroShift node + value: + interfaces: + - name: eth0 + type: ethernet + state: up + mac-address: {get_attr: [microshift0-machine-port, mac_address]} + ipv4: + enabled: true + dhcp: true + ipv6: + enabled: false + - name: eth1 + type: ethernet + state: up + mac-address: {get_attr: [microshift0-ctlplane-trunk-parent-port, mac_address]} + ipv4: + enabled: false + ipv6: + enabled: false + - name: eth2 + type: ethernet + state: up + mac-address: {get_attr: [microshift0-ironic-port, mac_address]} + ipv4: + enabled: false + ipv6: + enabled: false + - name: internalapi + type: vlan + state: up + vlan: + base-iface: eth1 + id: 20 + ipv4: + enabled: true + address: + - ip: 172.17.0.10 + prefix-length: 24 + ipv6: + enabled: false + - name: storage + type: vlan + state: up + vlan: + base-iface: eth1 + id: 21 + ipv4: + enabled: true + address: + - ip: 172.18.0.10 + prefix-length: 24 + ipv6: + enabled: false + - name: tenant + type: vlan + state: up + vlan: + base-iface: eth1 + id: 22 + ipv4: + enabled: true + address: + - ip: 172.19.0.10 + prefix-length: 24 + ipv6: + enabled: false + - name: ospbr + type: linux-bridge + state: up + bridge: + options: + stp: + enabled: false + port: + - name: eth1 + ipv4: + enabled: true + address: + - ip: 192.168.122.10 + prefix-length: 24 + ipv6: + enabled: false + - name: ironic + type: linux-bridge + state: up + bridge: + options: + stp: + enabled: false + port: + - name: eth2 + ipv4: + enabled: true + address: + - ip: 172.20.1.10 + prefix-length: 24 + ipv6: + enabled: false + + sushy_emulator_uuids: + description: UUIDs of instances to manage with sushy-tools - RedFish virtual BMC + value: + ironic0: {get_resource: ironic0} + ironic1: {get_resource: ironic1} + + ironic_nodes: + description: Ironic nodes YAML, used with openstack baremetal create to enroll nodes in Openstack Ironic + value: + nodes: + - name: ironic0 + driver: redfish + bios_interface: no-bios + boot_interface: redfish-virtual-media + driver_info: + redfish_address: http://sushy-emulator.apps.microshift.openstack.lab + redfish_system_id: + str_replace: + template: "/redfish/v1/Systems/$SYS_ID" + params: + $SYS_ID: {get_resource: ironic0} + redfish_username: admin + redfish_password: password + ports: + - address: {get_attr: [ironic0-port, mac_address]} + physical_network: ironic + - name: ironic1 + driver: redfish + bios_interface: no-bios + boot_interface: redfish-virtual-media + driver_info: + redfish_address: http://sushy-emulator.apps.microshift.openstack.lab + redfish_system_id: + str_replace: + template: "/redfish/v1/Systems/$SYS_ID" + params: + $SYS_ID: {get_resource: ironic1} + redfish_username: admin + redfish_password: password + ports: + - address: {get_attr: [ironic1-port, mac_address]} + physical_network: ironic + + controller_ansible_host: + description: > + Controller ansible host, this struct can be passed to the ansible.builtin.add_host module + value: + name: controller-0 + ansible_ssh_user: zuul + ansible_host: {get_attr: [controller-floating-ip, floating_ip_address]} + ansible_ssh_common_args: '-o StrictHostKeyChecking=no' + groups: + - controllers + + microshift_ansible_host: + description: > + MicroShift ansible host, this struct can be passed to the ansible.builtin.add_host module. + Uses controller-0 as SSH jump host (bastion) for access. + value: + name: microshift-0 + ansible_ssh_user: zuul + ansible_host: {get_attr: [microshift0-machine-port, fixed_ips, 0, ip_address]} + ansible_ssh_common_args: + str_replace: + template: '-o StrictHostKeyChecking=no -o ProxyJump=zuul@$controller_ip' + params: + $controller_ip: {get_attr: [controller-floating-ip, floating_ip_address]} + groups: + - microshifts diff --git a/scenarios/microshift-2-bm/manifests/control-plane/control-plane.yaml b/scenarios/microshift-2-bm/manifests/control-plane/control-plane.yaml new file mode 100644 index 00000000..1e9866b3 --- /dev/null +++ b/scenarios/microshift-2-bm/manifests/control-plane/control-plane.yaml @@ -0,0 +1,471 @@ +--- +apiVersion: v1 +data: + server-ca-passphrase: MTIzNDU2Nzg= +kind: Secret +metadata: + name: octavia-ca-passphrase + namespace: openstack +type: Opaque +--- +apiVersion: v1 +data: + AdminPassword: MTIzNDU2Nzg= + AodhDatabasePassword: MTIzNDU2Nzg= + AodhPassword: MTIzNDU2Nzg= + BarbicanDatabasePassword: MTIzNDU2Nzg= + BarbicanPassword: MTIzNDU2Nzg= + BarbicanSimpleCryptoKEK: r0wDZ1zrD5upafX9RDfYqvDkW2LENBWH7Gz9+Tr3NdM= + CeilometerPassword: MTIzNDU2Nzg= + CinderDatabasePassword: MTIzNDU2Nzg= + CinderPassword: MTIzNDU2Nzg= + DatabasePassword: MTIzNDU2Nzg= + DbRootPassword: MTIzNDU2Nzg= + DesignateDatabasePassword: MTIzNDU2Nzg= + DesignatePassword: MTIzNDU2Nzg= + GlanceDatabasePassword: MTIzNDU2Nzg= + GlancePassword: MTIzNDU2Nzg= + HeatAuthEncryptionKey: NzY3YzNlZDA1NmNiYWEzYjlkZmVkYjhjNmY4MjViZjA= + HeatDatabasePassword: MTIzNDU2Nzg= + HeatPassword: MTIzNDU2Nzg= + IronicDatabasePassword: MTIzNDU2Nzg= + IronicInspectorDatabasePassword: MTIzNDU2Nzg= + IronicInspectorPassword: MTIzNDU2Nzg= + IronicPassword: MTIzNDU2Nzg= + KeystoneDatabasePassword: MTIzNDU2Nzg= + ManilaDatabasePassword: MTIzNDU2Nzg= + ManilaPassword: MTIzNDU2Nzg= + MetadataSecret: MTIzNDU2Nzg0Mg== + NeutronDatabasePassword: MTIzNDU2Nzg= + NeutronPassword: MTIzNDU2Nzg= + NovaAPIDatabasePassword: MTIzNDU2Nzg= + NovaCell0DatabasePassword: MTIzNDU2Nzg= + NovaCell1DatabasePassword: MTIzNDU2Nzg= + NovaPassword: MTIzNDU2Nzg= + OctaviaDatabasePassword: MTIzNDU2Nzg= + OctaviaHeartbeatKey: MTIzNDU2Nzg= + OctaviaPassword: MTIzNDU2Nzg= + PlacementDatabasePassword: MTIzNDU2Nzg= + PlacementPassword: MTIzNDU2Nzg= + SwiftPassword: MTIzNDU2Nzg= +kind: Secret +metadata: + name: osp-secret + namespace: openstack +type: Opaque +--- +apiVersion: core.openstack.org/v1beta1 +kind: OpenStackControlPlane +metadata: + name: controlplane + namespace: openstack +spec: + barbican: + enabled: false + ceilometer: + enabled: false + cinder: + apiOverride: + route: + haproxy.router.openshift.io/timeout: 60s + template: + cinderAPI: + override: + service: + internal: + metadata: + annotations: + metallb.universe.tf/address-pool: internalapi + metallb.universe.tf/allow-shared-ip: internalapi + metallb.universe.tf/loadBalancerIPs: 172.17.0.80 + spec: + type: LoadBalancer + replicas: 1 + cinderBackup: + customServiceConfig: | + [DEFAULT] + backup_driver = cinder.backup.drivers.swift.SwiftBackupDriver + networkAttachments: + - storage + replicas: 1 + cinderScheduler: + replicas: 1 + cinderVolumes: + lvm-iscsi: + customServiceConfig: | + [lvm] + image_volume_cache_enabled = false + volume_driver = cinder.volume.drivers.lvm.LVMVolumeDriver + volume_group = cinder-volumes + target_protocol = iscsi + target_helper = lioadm + volume_backend_name = lvm_iscsi + target_ip_address=172.18.0.10 + target_secondary_ip_addresses = 172.19.0.10 + nodeSelector: + openstack.org/cinder-lvm: "" + replicas: 1 + customServiceConfig: | + # Debug logs by default, jobs can override as needed. + [DEFAULT] + debug = true + databaseInstance: openstack + preserveJobs: false + secret: osp-secret + uniquePodNames: true + designate: + enabled: false + dns: + template: + options: + - key: server + values: + - 192.168.32.254 + override: + service: + metadata: + annotations: + metallb.universe.tf/address-pool: ctlplane + metallb.universe.tf/allow-shared-ip: ctlplane + metallb.universe.tf/loadBalancerIPs: 192.168.122.80 + spec: + type: LoadBalancer + replicas: 1 + galera: + enabled: true + templates: + openstack: + replicas: 1 + secret: osp-secret + storageRequest: 5G + openstack-cell1: + replicas: 1 + secret: osp-secret + storageRequest: 5G + glance: + apiOverrides: + default: + route: + haproxy.router.openshift.io/timeout: 60s + template: + customServiceConfig: | + [DEFAULT] + debug = True + enabled_backends = default_backend:swift + + [glance_store] + default_backend = default_backend + + [default_backend] + swift_store_create_container_on_put = True + swift_store_auth_version = 3 + swift_store_auth_address = {{ .KeystoneInternalURL }} + swift_store_endpoint_type = internalURL + swift_store_user = service:glance + swift_store_key = {{ .ServicePassword }} + databaseInstance: openstack + glanceAPIs: + default: + networkAttachments: + - storage + override: + service: + internal: + metadata: + annotations: + metallb.universe.tf/address-pool: internalapi + metallb.universe.tf/allow-shared-ip: internalapi + metallb.universe.tf/loadBalancerIPs: 172.17.0.80 + spec: + type: LoadBalancer + replicas: 1 + preserveJobs: false + storage: + storageClass: lvms-local-storage + storageRequest: 10G + uniquePodNames: true + heat: + enabled: false + horizon: + enabled: false + ironic: + enabled: true + template: + databaseInstance: openstack + ironicAPI: + override: + service: + internal: + metadata: + annotations: + metallb.universe.tf/address-pool: ctlplane + metallb.universe.tf/allow-shared-ip: ctlplane + metallb.universe.tf/loadBalancerIPs: 192.168.122.80 + spec: + type: LoadBalancer + replicas: 1 + ironicConductors: + - customServiceConfig: | + [conductor] + power_state_change_timeout = 120 + + [redfish] + kernel_append_params = console=ttyS0 + + [neutron] + cleaning_network = provisioning + provisioning_network = provisioning + rescuing_network = provisioning + inspection_network = provisioning + networkAttachments: + - ironic + provisionNetwork: ironic + replicas: 1 + storageRequest: 10G + ironicInspector: + customServiceConfig: | + [capabilities] + boot_mode = true + + [processing] + update_pxe_enabled = false + inspectionNetwork: ironic + networkAttachments: + - ironic + override: + service: + internal: + metadata: + annotations: + metallb.universe.tf/address-pool: ctlplane + metallb.universe.tf/allow-shared-ip: ctlplane + metallb.universe.tf/loadBalancerIPs: 192.168.122.80 + spec: + type: LoadBalancer + preserveJobs: false + replicas: 1 + ironicNeutronAgent: + replicas: 1 + preserveJobs: false + rpcTransport: oslo + secret: osp-secret + keystone: + apiOverride: + route: {} + template: + databaseInstance: openstack + override: + service: + internal: + metadata: + annotations: + metallb.universe.tf/address-pool: internalapi + metallb.universe.tf/allow-shared-ip: internalapi + metallb.universe.tf/loadBalancerIPs: 172.17.0.80 + spec: + type: LoadBalancer + preserveJobs: false + replicas: 1 + secret: osp-secret + manila: + enabled: false + memcached: + templates: + memcached: + replicas: 1 + neutron: + apiOverride: + route: {} + template: + customServiceConfig: | + [DEFAULT] + vlan_transparent = true + agent_down_time = 600 + router_distributed = true + router_scheduler_driver = neutron.scheduler.l3_agent_scheduler.ChanceScheduler + allow_automatic_l3agent_failover = true + debug = true + + [agent] + report_interval = 300 + + [database] + max_retries = -1 + db_max_retries = -1 + + [keystone_authtoken] + region_name = regionOne + memcache_use_advanced_pool = True + + [oslo_messaging_notifications] + driver = noop + + [oslo_middleware] + enable_proxy_headers_parsing = true + + [oslo_policy] + policy_file = /etc/neutron/policy.yaml + + [ovs] + igmp_snooping_enable = true + + [ovn] + ovsdb_probe_interval = 60000 + ovn_emit_need_to_frag = true + + [ml2] + global_physnet_mtu = 1442 + type_drivers = geneve,vxlan,vlan,flat,local + tenant_network_types = geneve,flat + databaseInstance: openstack + networkAttachments: + - internalapi + override: + service: + internal: + metadata: + annotations: + metallb.universe.tf/address-pool: internalapi + metallb.universe.tf/allow-shared-ip: internalapi + metallb.universe.tf/loadBalancerIPs: 172.17.0.80 + spec: + type: LoadBalancer + preserveJobs: false + replicas: 1 + secret: osp-secret + nova: + apiOverride: + route: {} + template: + apiServiceTemplate: + override: + service: + internal: + metadata: + annotations: + metallb.universe.tf/address-pool: internalapi + metallb.universe.tf/allow-shared-ip: internalapi + metallb.universe.tf/loadBalancerIPs: 172.17.0.80 + spec: + type: LoadBalancer + replicas: 1 + cellTemplates: + cell0: + cellDatabaseAccount: nova-cell0 + cellDatabaseInstance: openstack + cellMessageBusInstance: rabbitmq + hasAPIAccess: true + cell1: + cellDatabaseAccount: nova-cell1 + cellDatabaseInstance: openstack-cell1 + cellMessageBusInstance: rabbitmq-cell1 + hasAPIAccess: true + novaComputeTemplates: + compute-ironic: + computeDriver: ironic.IronicDriver + metadataServiceTemplate: + override: + service: + metadata: + annotations: + metallb.universe.tf/address-pool: internalapi + metallb.universe.tf/allow-shared-ip: internalapi + metallb.universe.tf/loadBalancerIPs: 172.17.0.80 + spec: + type: LoadBalancer + replicas: 1 + preserveJobs: false + schedulerServiceTemplate: + replicas: 1 + secret: osp-secret + octavia: + enabled: false + ovn: + template: + ovnController: + networkAttachment: tenant + nicMappings: + datacentre: ocpbr + ironic: ironic + ovnDBCluster: + ovndbcluster-nb: + dbType: NB + networkAttachment: internalapi + replicas: 1 + storageRequest: 10G + ovndbcluster-sb: + dbType: SB + networkAttachment: internalapi + replicas: 1 + storageRequest: 10G + ovnNorthd: + logLevel: info + nThreads: 1 + replicas: 1 + resources: {} + tls: {} + placement: + apiOverride: + route: {} + template: + databaseInstance: openstack + override: + service: + internal: + metadata: + annotations: + metallb.universe.tf/address-pool: internalapi + metallb.universe.tf/allow-shared-ip: internalapi + metallb.universe.tf/loadBalancerIPs: 172.17.0.80 + spec: + type: LoadBalancer + preserveJobs: false + replicas: 1 + secret: osp-secret + rabbitmq: + templates: + rabbitmq: + override: + service: + metadata: + annotations: + metallb.universe.tf/address-pool: internalapi + metallb.universe.tf/loadBalancerIPs: 172.17.0.85 + spec: + type: LoadBalancer + replicas: 1 + rabbitmq-cell1: + override: + service: + metadata: + annotations: + metallb.universe.tf/address-pool: internalapi + metallb.universe.tf/loadBalancerIPs: 172.17.0.86 + spec: + type: LoadBalancer + replicas: 1 + secret: osp-secret + storageClass: lvms-local-storage + swift: + enabled: true + proxyOverride: + route: {} + template: + swiftProxy: + override: + service: + internal: + metadata: + annotations: + metallb.universe.tf/address-pool: internalapi + metallb.universe.tf/allow-shared-ip: internalapi + metallb.universe.tf/loadBalancerIPs: 172.17.0.80 + spec: + type: LoadBalancer + replicas: 1 + swiftRing: + ringReplicas: 1 + swiftStorage: + replicas: 1 + telemetry: + enabled: false diff --git a/scenarios/microshift-2-bm/manifests/control-plane/networking/metallb.yaml b/scenarios/microshift-2-bm/manifests/control-plane/networking/metallb.yaml new file mode 100644 index 00000000..a71f9d48 --- /dev/null +++ b/scenarios/microshift-2-bm/manifests/control-plane/networking/metallb.yaml @@ -0,0 +1,88 @@ +--- +apiVersion: metallb.io/v1beta1 +kind: IPAddressPool +metadata: + labels: + osp/lb-addresses-type: standard + name: ctlplane + namespace: metallb-system +spec: + addresses: + - 192.168.122.80-192.168.122.90 +--- +apiVersion: metallb.io/v1beta1 +kind: IPAddressPool +metadata: + labels: + osp/lb-addresses-type: standard + name: internalapi + namespace: metallb-system +spec: + addresses: + - 172.17.0.80-172.17.0.90 +--- +apiVersion: metallb.io/v1beta1 +kind: IPAddressPool +metadata: + labels: + osp/lb-addresses-type: standard + name: storage + namespace: metallb-system +spec: + addresses: + - 172.18.0.80-172.18.0.90 +--- +apiVersion: metallb.io/v1beta1 +kind: IPAddressPool +metadata: + labels: + osp/lb-addresses-type: standard + name: tenant + namespace: metallb-system +spec: + addresses: + - 172.19.0.80-172.19.0.90 +--- +apiVersion: metallb.io/v1beta1 +kind: L2Advertisement +metadata: + name: ctlplane + namespace: metallb-system +spec: + interfaces: + - ospbr + ipAddressPools: + - ctlplane +--- +apiVersion: metallb.io/v1beta1 +kind: L2Advertisement +metadata: + name: internalapi + namespace: metallb-system +spec: + interfaces: + - internalapi + ipAddressPools: + - internalapi +--- +apiVersion: metallb.io/v1beta1 +kind: L2Advertisement +metadata: + name: storage + namespace: metallb-system +spec: + interfaces: + - storage + ipAddressPools: + - storage +--- +apiVersion: metallb.io/v1beta1 +kind: L2Advertisement +metadata: + name: tenant + namespace: metallb-system +spec: + interfaces: + - tenant + ipAddressPools: + - tenant diff --git a/scenarios/microshift-2-bm/manifests/control-plane/networking/nad.yaml b/scenarios/microshift-2-bm/manifests/control-plane/networking/nad.yaml new file mode 100644 index 00000000..fa5e8f71 --- /dev/null +++ b/scenarios/microshift-2-bm/manifests/control-plane/networking/nad.yaml @@ -0,0 +1,134 @@ +--- +apiVersion: k8s.cni.cncf.io/v1 +kind: NetworkAttachmentDefinition +metadata: + labels: + osp/net: ctlplane + osp/net-attach-def-type: standard + name: ctlplane + namespace: openstack +spec: + config: | + { + "cniVersion": "0.3.1", + "name": "ctlplane", + "type": "macvlan", + "master": "ospbr", + "ipam": { + "type": "whereabouts", + "range": "192.168.122.0/24", + "range_start": "192.168.122.30", + "range_end": "192.168.122.70" + } + } +--- +apiVersion: k8s.cni.cncf.io/v1 +kind: NetworkAttachmentDefinition +metadata: + labels: + osp/net: ironic + osp/net-attach-def-type: standard + name: ironic + namespace: openstack +spec: + config: | + { + "cniVersion": "0.3.1", + "name": "ironic", + "type": "bridge", + "bridge": "ironic", + "mtu": 1442, + "ipam": { + "type": "whereabouts", + "range": "172.20.1.0/24", + "range_start": "172.20.1.30", + "range_end": "172.20.1.70" + } + } +--- +apiVersion: k8s.cni.cncf.io/v1 +kind: NetworkAttachmentDefinition +metadata: + labels: + osp/net: datacentre + osp/net-attach-def-type: standard + name: datacentre + namespace: openstack +spec: + config: | + { + "cniVersion": "0.3.1", + "name": "datacentre", + "type": "bridge", + "bridge": "ospbr", + "ipam": {} + } +--- +apiVersion: k8s.cni.cncf.io/v1 +kind: NetworkAttachmentDefinition +metadata: + labels: + osp/net: internalapi + osp/net-attach-def-type: standard + name: internalapi + namespace: openstack +spec: + config: | + { + "cniVersion": "0.3.1", + "name": "internalapi", + "type": "macvlan", + "master": "internalapi", + "ipam": { + "type": "whereabouts", + "range": "172.17.0.0/24", + "range_start": "172.17.0.30", + "range_end": "172.17.0.70" + } + } +--- +apiVersion: k8s.cni.cncf.io/v1 +kind: NetworkAttachmentDefinition +metadata: + labels: + osp/net: storage + osp/net-attach-def-type: standard + name: storage + namespace: openstack +spec: + config: | + { + "cniVersion": "0.3.1", + "name": "storage", + "type": "macvlan", + "master": "storage", + "ipam": { + "type": "whereabouts", + "range": "172.18.0.0/24", + "range_start": "172.18.0.30", + "range_end": "172.18.0.70" + } + } +--- +apiVersion: k8s.cni.cncf.io/v1 +kind: NetworkAttachmentDefinition +metadata: + labels: + osp/net: tenant + osp/net-attach-def-type: standard + name: tenant + namespace: openstack +spec: + config: | + { + "cniVersion": "0.3.1", + "name": "tenant", + "type": "macvlan", + "master": "tenant", + "ipam": { + "type": "whereabouts", + "range": "172.19.0.0/24", + "range_start": "172.19.0.30", + "range_end": "172.19.0.70" + } + } diff --git a/scenarios/microshift-2-bm/manifests/control-plane/networking/nncp.yaml b/scenarios/microshift-2-bm/manifests/control-plane/networking/nncp.yaml new file mode 100644 index 00000000..4711ea8d --- /dev/null +++ b/scenarios/microshift-2-bm/manifests/control-plane/networking/nncp.yaml @@ -0,0 +1,130 @@ +--- +apiVersion: nmstate.io/v1 +kind: NodeNetworkConfigurationPolicy +metadata: + labels: + osp/nncm-config-type: standard + name: microshift-0 + namespace: openstack +spec: + desiredState: + dns-resolver: + config: + search: [] + server: + - 192.168.32.254 + interfaces: + - name: internalapi + type: vlan + description: internalapi vlan interface + ipv4: + address: + - ip: 172.17.0.10 + prefix-length: "24" + dhcp: false + enabled: true + ipv6: + enabled: false + mtu: 1442 + state: up + vlan: + base-iface: eth1 + id: "20" + - name: storage + type: vlan + description: storage vlan interface + ipv4: + address: + - ip: 172.18.0.10 + prefix-length: "24" + dhcp: false + enabled: true + ipv6: + enabled: false + mtu: 1442 + state: up + vlan: + base-iface: eth1 + id: "21" + - name: tenant + type: vlan + description: tenant vlan interface + ipv4: + address: + - ip: 172.19.0.10 + prefix-length: "24" + dhcp: false + enabled: true + ipv6: + enabled: false + mtu: 1442 + state: up + vlan: + base-iface: eth1 + id: "22" + - description: ctlplane interface + mtu: 1442 + name: eth1 + state: up + type: ethernet + - name: ospbr + type: linux-bridge + description: linux-bridge over ctlplane interface + bridge: + options: + stp: + enabled: false + port: + - name: eth1 + vlan: {} + ipv4: + address: + - ip: 192.168.122.10 + prefix-length: "24" + dhcp: false + enabled: true + ipv6: + enabled: false + mtu: 1442 + state: up + - name: ironic + type: linux-bridge + description: Ironic bridge + bridge: + options: + stp: + enabled: false + port: + - name: eth2 + ipv4: + address: + - ip: 172.20.1.10 + prefix-length: "24" + enabled: true + ipv6: + enabled: false + mtu: 1442 + - name: ironicvrf + description: Ironic VRF + state: up + type: vrf + vrf: + port: + - ironic + route-table-id: 10 + route-rules: + config: [] + routes: + config: + - destination: 0.0.0.0/0 + metric: 150 + next-hop-address: 172.20.1.1 + next-hop-interface: ironic + table-id: 10 + - destination: 172.20.1.0/24 + metric: 150 + next-hop-address: 192.168.122.1 + next-hop-interface: ospbr + nodeSelector: + kubernetes.io/hostname: microshift-0 + node-role.kubernetes.io/worker: "" diff --git a/scenarios/microshift-2-bm/test-operator/README.md b/scenarios/microshift-2-bm/test-operator/README.md new file mode 100644 index 00000000..dc44ea00 --- /dev/null +++ b/scenarios/microshift-2-bm/test-operator/README.md @@ -0,0 +1,42 @@ + +# AI generated README + +## Tempest Tests Configuration for Test Operator + +The YAML file, `tempest-tests.yml`, is a configuration for running Tempest +tests, which is a validation framework for OpenStack. Here's a breakdown of the +configuration: + +1. **apiVersion, kind, and metadata**: These fields define the API version, + kind (type) of resource, and metadata (name and namespace) for the Tempest + test job. + +2. **spec**: This section contains the configuration for the Tempest test job. + + - **networkAttachments**: This field specifies the network attachment for + the test job. In this case, it's set to `ctlplane`. + - **storageClass**: This field sets the storage class for the test job to + `lvms-local-storage`. + - **privileged**: This field is set to `true`, which means the test + containers will have elevated privileges. + - **workflow**: This section defines the steps to be executed in the test + job. There are two steps in this configuration: + - **ironic-scenario-testing**: This step runs scenario tests for Ironic, + the OpenStack bare-metal provisioning service. The `tempestconfRun` + section configures Tempest settings for this step, such as disabling + isolated networks, setting the number of available nodes, and specifying + the compute flavor and hypervisor type. The `tempestRun` section + specifies the concurrency level and the list of tests to include and + exclude. + - **ironic-api-testing**: This step runs API tests for Ironic. Similar to + the previous step, the `tempestconfRun` section configures Tempest + settings, and the `tempestRun` section specifies the concurrency level + and the list of tests to include and exclude. + +In summary, this YAML file configures a Tempest test job to run two types of +tests for Ironic: scenario tests and API tests. The tests are executed with +specific configurations and concurrency levels. diff --git a/scenarios/microshift-2-bm/test-operator/automation-vars.yml b/scenarios/microshift-2-bm/test-operator/automation-vars.yml new file mode 100644 index 00000000..5a743310 --- /dev/null +++ b/scenarios/microshift-2-bm/test-operator/automation-vars.yml @@ -0,0 +1,380 @@ +--- +stages: + - name: Apply ironic network-attachement-definition + documentation: >- + Creates a Multus NetworkAttachmentDefinition that allows pods to attach to the ironic + provisioning network. This is required for sushy-emulator to communicate with baremetal + nodes during provisioning operations via the dedicated provisioning network. + manifest: manifests/nad.yaml + wait_conditions: + - >- + oc wait -n sushy-emulator network-attachment-definitions.k8s.cni.cncf.io ironic + --for jsonpath='{.metadata.annotations}' --timeout=30s + + - name: Patch RedFish Sushy Emulator Deployment - add network attachment + documentation: >- + Modifies the sushy-emulator deployment to attach to the ironic provisioning network + via CNI network annotation. This enables the BMC simulator to receive Redfish API + calls on the correct network segment where baremetal nodes expect their BMC interfaces. + shell: | + set -xe -o pipefail + + TMP_DIR="$(mktemp -d)" + trap 'rm -rf -- "$TMP_DIR"' EXIT + + oc project sushy-emulator + + cat << EOF > ${TMP_DIR}/sushy-emulator-network-annotations-patch.yaml + spec: + template: + metadata: + annotations: + k8s.v1.cni.cncf.io/networks: '[{"name":"ironic","namespace":"sushy-emulator","interface":"ironic"}]' + EOF + + oc patch deployments.apps sushy-emulator --patch-file ${TMP_DIR}/sushy-emulator-network-annotations-patch.yaml + wait_conditions: + - "oc -n sushy-emulator wait deployments.apps sushy-emulator --for condition=Available --timeout=300s" + + - name: Set a multiattach volume type and create it if needed + documentation: >- + Creates a Cinder volume type with multiattach capability enabled, allowing volumes + to be simultaneously attached to multiple instances. This is essential for shared + storage scenarios like clustered databases or file systems in the test environment. + shell: | + set -xe -o pipefail + oc project openstack + + oc rsh openstackclient openstack volume type show multiattach &>/dev/null || \ + oc rsh openstackclient openstack volume type create multiattach + + oc rsh openstackclient openstack volume type set --property multiattach=" True" multiattach + + - name: Create public network if needed + documentation: >- + Establishes the external network that provides floating IP connectivity to instances. + This network is mapped to the physical 'datacentre' network and serves as the gateway + for external traffic routing. Essential for instances to reach external services and + for external clients to access instance services via floating IPs. + shell: | + set -xe -o pipefail + oc project openstack + + oc rsh openstackclient openstack network show public &>/dev/null || \ + oc rsh openstackclient openstack network create public \ + --external \ + --no-share \ + --default \ + --provider-network-type flat \ + --provider-physical-network datacentre + + - name: Create subnet on public network if needed + documentation: >- + Defines the IP address pool and routing configuration for the public network. + The 192.168.122.0/24 range with allocation pool 171-250 provides floating IPs + while preserving lower addresses for infrastructure. Gateway 192.168.122.1 + routes traffic to external networks. + shell: | + set -xe -o pipefail + oc project openstack + + oc rsh openstackclient openstack subnet show public_subnet &>/dev/null || \ + oc rsh openstackclient openstack subnet create public_subnet \ + --network public \ + --subnet-range 192.168.122.0/24 \ + --allocation-pool start=192.168.122.171,end=192.168.122.250 \ + --gateway 192.168.122.1 \ + --dhcp + + - name: Create private network if needed + documentation: >- + Creates the default tenant network for instance-to-instance communication. + This shared network allows multiple tenants to deploy instances that can + communicate privately while being isolated from external networks until + floating IPs are assigned. + shell: | + set -xe -o pipefail + oc project openstack + + oc rsh openstackclient openstack network show private &>/dev/null || \ + oc rsh openstackclient openstack network create private --share + + - name: Create subnet on private network if needed + documentation: >- + Configures the private tenant network with RFC1918 addressing (10.2.0.0/24). + This subnet provides DHCP-assigned IP addresses for instances deployed on + the private network, enabling inter-instance communication before floating + IP assignment for external access. + shell: | + set -xe -o pipefail + oc project openstack + + oc rsh openstackclient openstack subnet show private_subnet &>/dev/null || \ + oc rsh openstackclient openstack subnet create private_subnet \ + --network private \ + --subnet-range 10.2.0.0/24 \ + --allocation-pool start=10.2.0.10,end=10.2.0.250 \ + --gateway 10.2.0.1 \ + --dhcp + + - name: Create network for ironic provisioning if needed + documentation: >- + Establishes the dedicated baremetal provisioning network mapped to the physical + 'ironic' network interface. This isolated network carries PXE boot traffic, DHCP + for baremetal nodes, and communication between Ironic services and nodes during + deployment operations, keeping provisioning traffic separate from tenant networks. + shell: | + set -xe -o pipefail + oc project openstack + + oc rsh openstackclient openstack network show provisioning &>/dev/null || \ + oc rsh openstackclient \ + openstack network create provisioning \ + --share \ + --provider-physical-network ironic \ + --provider-network-type flat + + - name: Create subnet for ironic provisioning if needed + documentation: >- + Configures IP addressing for the baremetal provisioning network (172.20.1.0/24). + DNS server 192.168.122.80 provides name resolution during node deployment. + The allocation pool 100-200 reserves addresses for DHCP assignment to baremetal + nodes during their provisioning lifecycle. + shell: | + set -xe -o pipefail + oc project openstack + + oc rsh openstackclient openstack subnet show provisioning-subnet &>/dev/null || \ + oc rsh openstackclient \ + openstack subnet create provisioning-subnet \ + --network provisioning \ + --subnet-range 172.20.1.0/24 \ + --gateway 172.20.1.1 \ + --dns-nameserver 192.168.122.80 \ + --allocation-pool start=172.20.1.100,end=172.20.1.200 + + - name: Create baremetal flavor if needed + documentation: >- + Defines Nova flavor for baremetal instances with custom resource requirements. + Uses CUSTOM_BAREMETAL=1 to match against baremetal node resource classes, + while setting standard resources (VCPU, MEMORY_MB, DISK_GB) to 0 since + physical resources are managed by Ironic rather than Nova's scheduler. + UEFI boot mode ensures compatibility with modern baremetal hardware. + shell: | + set -xe -o pipefail + oc project openstack + + oc rsh openstackclient openstack flavor show baremetal &>/dev/null || \ + oc rsh openstackclient \ + openstack flavor create baremetal \ + --id 123456789-1234-1234-1234-000000000001 \ + --ram 1024 \ + --vcpus 1 \ + --disk 15 \ + --property resources:VCPU=0 \ + --property resources:MEMORY_MB=0 \ + --property resources:DISK_GB=0 \ + --property resources:CUSTOM_BAREMETAL=1 \ + --property capabilities:boot_mode=uefi + + - name: Copy ironic_nodes.yaml to the openstackclient pod + documentation: >- + Transfers the baremetal node definition file containing BMC credentials, + hardware specifications, and network configurations from the local filesystem + to the OpenStack client pod. This file defines the physical infrastructure + that Ironic will manage for baremetal provisioning. + shell: | + set -xe -o pipefail + oc project openstack + oc cp ~/data/ironic_nodes.yaml openstackclient:ironic_nodes.yaml + + - name: Enroll nodes in ironic + documentation: >- + Registers physical baremetal nodes with the Ironic service using the node + definitions from ironic_nodes.yaml. This creates Ironic node records with + BMC access credentials, hardware profiles, and port configurations, marking + the beginning of the node lifecycle management in OpenStack. + shell: | + set -xe -o pipefail + oc project openstack + oc rsh openstackclient openstack baremetal create ironic_nodes.yaml + + - name: Wait for ironic nodes to get to state - enroll + documentation: >- + Monitors node state transition to 'enroll' status, indicating that Ironic + has successfully registered the nodes and validated basic BMC connectivity. + This is the first state in the baremetal provisioning lifecycle, confirming + that nodes are recognized by the system before management operations begin. + shell: | + oc project openstack + + counter=0 + max_retries=100 + node_state=enroll + until ! oc rsh openstackclient openstack baremetal node list -f value -c "Provisioning State" | grep -P "^(?!${node_state}).*$"; do + ((counter++)) + if (( counter > max_retries )); then + echo "ERROR: Timeout. Nodes did not reach state: enroll" + exit 1 + fi + echo "Waiting for nodes to reach state enroll" + sleep 10 + done + + - name: Manage ironic nodes + documentation: >- + Initiates the transition from 'enroll' to 'manageable' state by instructing + Ironic to perform hardware introspection and validation. During this process, + Ironic will power on nodes, inspect hardware capabilities, and prepare them + for provisioning operations while validating BMC access and power management. + shell: | + set -xe -o pipefail + oc project openstack + + oc rsh openstackclient openstack baremetal node manage ironic0 + oc rsh openstackclient openstack baremetal node manage ironic1 + + - name: Wait for ironic nodes to get to state - manageable + documentation: >- + Waits for nodes to complete hardware introspection and reach 'manageable' state. + In this state, Ironic has successfully inventoried hardware resources, validated + BMC functionality, and confirmed the nodes are ready for cleaning and provisioning + operations. This is a prerequisite for making nodes available to tenants. + shell: | + oc project openstack + + counter=0 + max_retries=100 + node_state=manageable + until ! oc rsh openstackclient openstack baremetal node list -f value -c "Provisioning State" | grep -P "^(?!${node_state}).*$"; do + ((counter++)) + if (( counter > max_retries )); then + echo "ERROR: Timeout. Nodes did not reach state: manageable" + exit 1 + fi + echo "Waiting for nodes to reach state manageable" + sleep 10 + done + + - name: Power off the ironic nodes + documentation: >- + Ensures all baremetal nodes are powered down before configuration changes. + This prevents potential issues during capability updates and ensures a clean + state before transitioning nodes to 'available'. Power management validation + also confirms BMC functionality is working correctly. + shell: | + set -xe -o pipefail + oc project openstack + + oc rsh openstackclient openstack baremetal node power off ironic0 + oc rsh openstackclient openstack baremetal node power off ironic1 + + - name: Ensure ironic nodes are powered off + documentation: >- + Verifies that power management commands have taken effect and all nodes + report 'power off' status. This confirmation is critical before transitioning + to 'available' state, as Nova expects baremetal nodes to be powered off + when not actively hosting instances. + shell: | + oc project openstack + + counter=0 + max_retries=100 + power_state="off" + until ! oc rsh openstackclient openstack baremetal node list -f value -c "Power State" | grep -P "^power.(?!${power_state}).*$"; do + ((counter++)) + if (( counter > max_retries )); then + echo "ERROR: Timeout. Nodes did not reach power state: power off" + exit 1 + fi + echo "Waiting for nodes to reach power state off" + sleep 10 + done + + - name: Set capabilities boot_mode:uefi for ironic nodes + documentation: >- + Configures nodes to use UEFI boot mode instead of legacy BIOS. This capability + setting influences how Ironic prepares boot images and configures network boot + options. UEFI boot is required for modern security features like Secure Boot + and provides better hardware compatibility for contemporary systems. + shell: | + set -xe -o pipefail + oc project openstack + + oc rsh openstackclient openstack baremetal node set --property capabilities='boot_mode:uefi' ironic0 + oc rsh openstackclient openstack baremetal node set --property capabilities='boot_mode:uefi' ironic1 + + - name: Provide ironic nodes + documentation: >- + Transitions nodes from 'manageable' to 'available' state, making them eligible + for tenant provisioning. This triggers automated cleaning processes to ensure + nodes are in a pristine state, removing any residual data from previous + deployments and preparing them for new instance launches. + shell: | + set -xe -o pipefail + oc project openstack + + oc rsh openstackclient openstack baremetal node provide ironic0 + oc rsh openstackclient openstack baremetal node provide ironic1 + + - name: Wait for ironic nodes to get to state - available + documentation: >- + Confirms nodes have completed the cleaning process and reached 'available' state. + Available nodes appear in Nova's resource inventory and can be allocated to + instance requests that match the baremetal flavor. This state indicates the + baremetal infrastructure is fully operational and ready for workload deployment. + shell: | + oc project openstack + + counter=0 + max_retries=100 + node_state=available + until ! oc rsh openstackclient openstack baremetal node list -f value -c "Provisioning State" | grep -P "^(?!${node_state}).*$"; do + ((counter++)) + if (( counter > max_retries )); then + echo "ERROR: Timeout. Nodes did not reach state: available" + exit 1 + fi + echo "Waiting for nodes to reach state: available" + sleep 10 + done + + - name: Wait for expected compute services (OSPRH-10942) + documentation: >- + Waits for Nova compute services to register and become available in the + Nova service registry. The hotstack-nova-discover-hosts utility ensures + that Ironic conductor services are properly registered as compute nodes, + enabling Nova scheduler to place baremetal instances. References bug + OSPRH-10942 related to compute service discovery timing issues. + wait_conditions: + - >- + timeout --foreground 5m hotstack-nova-discover-hosts + --namespace openstack --num-computes 1 + + - name: Run tempest + documentation: >- + Executes comprehensive OpenStack validation tests using the Tempest framework. + These tests validate API functionality, resource management, and integration + between OpenStack services in the baremetal environment. Includes both the + test execution phase (workflowStep=0) and log collection phase (workflowStep=1) + to ensure complete validation results are captured for analysis. + manifest: tempest-tests.yml + wait_conditions: + - >- + oc wait -n openstack tempests.test.openstack.org tempest-tests + --for condition=ServiceConfigReady --timeout=120s + wait_pod_completion: + - namespace: openstack + labels: + operator: test-operator + service: tempest + workflowStep: "0" + timeout: 900 + poll_interval: 15 + - namespace: openstack + labels: + operator: test-operator + service: tempest + workflowStep: "1" + timeout: 900 + poll_interval: 15 diff --git a/scenarios/microshift-2-bm/test-operator/manifests/nad.yaml b/scenarios/microshift-2-bm/test-operator/manifests/nad.yaml new file mode 100644 index 00000000..f10e0061 --- /dev/null +++ b/scenarios/microshift-2-bm/test-operator/manifests/nad.yaml @@ -0,0 +1,21 @@ +--- +apiVersion: k8s.cni.cncf.io/v1 +kind: NetworkAttachmentDefinition +metadata: + name: ironic + namespace: sushy-emulator +spec: + config: | + { + "cniVersion": "0.3.1", + "name": "ironic", + "type": "bridge", + "bridge": "ironic", + "mtu": 1442, + "ipam": { + "type": "whereabouts", + "range": "172.20.1.0/24", + "range_start": "172.20.1.71", + "range_end": "172.20.1.75" + } + } diff --git a/scenarios/microshift-2-bm/test-operator/tempest-tests.yml b/scenarios/microshift-2-bm/test-operator/tempest-tests.yml new file mode 100644 index 00000000..33195494 --- /dev/null +++ b/scenarios/microshift-2-bm/test-operator/tempest-tests.yml @@ -0,0 +1,92 @@ +--- +apiVersion: test.openstack.org/v1beta1 +kind: Tempest +metadata: + name: tempest-tests + namespace: openstack +spec: + networkAttachments: + - ctlplane + privileged: true + workflow: + - stepName: ironic-scenario-testing + storageClass: lvms-local-storage + tempestconfRun: + create: true + overrides: | + auth.create_isolated_networks false + baremetal.available_nodes 2 + baremetal.max_microversion 1.82 + compute-feature-enabled.disk_config false + compute-feature-enabled.interface_attach false + compute.fixed_network_name provisioning + compute.flavor_ref 123456789-1234-1234-1234-000000000001 + compute.hypervisor_type ironic + compute.build_timeout 900 + network.shared_physical_network true + service_available.ironic_inspector false + service_available.ironic true + service_available.murano false + validation.connect_method fixed + validation.network_for_ssh provisioning + tempestRun: + concurrency: 4 + includeList: | + ^ironic_tempest_plugin.tests.scenario.test_baremetal_basic_ops.* + excludeList: | + ^ironic_tempest_plugin.tests.scenario.test_baremetal_basic_ops.BaremetalBasicOps.test_baremetal_server_ops_partition_image + + - stepName: ironic-api-testing + storageClass: lvms-local-storage + tempestconfRun: + create: true + overrides: | + baremetal.driver fake-hardware + baremetal.max_microversion 1.82 + service_available.ironic_inspector true + service_available.ironic true + service_available.murano false + tempestRun: + concurrency: 8 + includeList: | + ^ironic_tempest_plugin.tests.api.* + excludeList: | + ^ironic_tempest_plugin.tests.api.admin.test_allocations.TestAllocations.test_create_allocation_candidate_node + ^ironic_tempest_plugin.tests.api.admin.test_allocations.TestAllocations.test_create_allocation_node_mismatch + ^ironic_tempest_plugin.tests.api.admin.test_allocations.TestAllocations.test_create_allocation_resource_class_mismatch + ^ironic_tempest_plugin.tests.api.admin.test_allocations.TestAllocations.test_create_allocation_traits_mismatch + ^ironic_tempest_plugin.tests.api.admin.test_allocations.TestAllocations.test_create_allocation_with_traits + ^ironic_tempest_plugin.tests.api.admin.test_allocations.TestAllocations.test_create_show_allocation + ^ironic_tempest_plugin.tests.api.admin.test_allocations.TestAllocations.test_delete_allocation + ^ironic_tempest_plugin.tests.api.admin.test_allocations.TestAllocations.test_delete_allocation_by_name + ^ironic_tempest_plugin.tests.api.admin.test_allocations.TestAllocations.test_list_allocations + ^ironic_tempest_plugin.tests.api.admin.test_allocations.TestAllocations.test_list_allocations_by_state + ^ironic_tempest_plugin.tests.api.admin.test_allocations.TestAllocations.test_show_by_name + ^ironic_tempest_plugin.tests.api.admin.test_allocations.TestBackfill.test_backfill_allocation + ^ironic_tempest_plugin.tests.api.admin.test_allocations.TestBackfill.test_backfill_without_resource_class + ^ironic_tempest_plugin.tests.api.admin.test_nodestates.TestNodeStatesV1_11.test_set_node_provision_state + ^ironic_tempest_plugin.tests.api.admin.test_nodestates.TestNodeStatesV1_1.test_set_node_provision_state + ^ironic_tempest_plugin.tests.api.admin.test_nodestates.TestNodeStatesV1_2.test_set_node_provision_state + ^ironic_tempest_plugin.tests.api.admin.test_nodestates.TestNodeStatesV1_4.test_set_node_provision_state + ^ironic_tempest_plugin.tests.api.admin.test_nodestates.TestNodeStatesV1_6.test_set_node_provision_state + ^ironic_tempest_plugin.tests.api.admin.test_nodes.TestHardwareInterfaces.test_reset_interfaces + ^ironic_tempest_plugin.tests.api.admin.test_nodes.TestNodeProtected.test_node_protected + ^ironic_tempest_plugin.tests.api.admin.test_nodes.TestNodeProtected.test_node_protected_from_deletion + ^ironic_tempest_plugin.tests.api.admin.test_nodes.TestNodeProtected.test_node_protected_negative + ^ironic_tempest_plugin.tests.api.admin.test_nodes.TestNodeProtected.test_node_protected_set_unset + ^ironic_tempest_plugin.tests.api.admin.test_nodes.TestNodesProtectedOldApi.test_node_protected_old_api + ^ironic_tempest_plugin.tests.api.admin.test_nodes.TestNodesVif.test_vif_already_attached_on_internal_info + ^ironic_tempest_plugin.tests.api.admin.test_nodes.TestNodesVif.test_vif_already_attached_with_portgroups + ^ironic_tempest_plugin.tests.api.admin.test_nodes.TestNodesVif.test_vif_already_set + ^ironic_tempest_plugin.tests.api.admin.test_nodes.TestNodesVif.test_vif_attach_no_args + ^ironic_tempest_plugin.tests.api.admin.test_nodes.TestNodesVif.test_vif_attach_no_free_port + ^ironic_tempest_plugin.tests.api.admin.test_nodes.TestNodesVif.test_vif_attach_no_port + ^ironic_tempest_plugin.tests.api.admin.test_nodes.TestNodesVif.test_vif_attach_port_not_in_portgroup + ^ironic_tempest_plugin.tests.api.admin.test_nodes.TestNodesVif.test_vif_attach_with_empty_portgroup + ^ironic_tempest_plugin.tests.api.admin.test_nodes.TestNodesVif.test_vif_detach_not_existing + ^ironic_tempest_plugin.tests.api.admin.test_nodes.TestNodesVif.test_vif_on_port + ^ironic_tempest_plugin.tests.api.admin.test_nodes.TestNodesVif.test_vif_on_portgroup + ^ironic_tempest_plugin.tests.api.admin.test_nodes.TestResetInterfaces.test_reset_interfaces + ^ironic_tempest_plugin.tests.api.rbac_defaults.test_nodes.TestNodeProjectReader.test_reader_cannot_update_owner_provisioned + ^ironic_tempest_plugin.tests.api.rbac_defaults.test_nodes.TestNodeSystemReader.test_reader_cannot_update_owner_provisioned + ^ironic_tempest_plugin.tests.api.rbac_defaults.test_nodes.TestNodeSystemReader.*