crowbar_framework/spec/models/api/upgrade_spec.rb
require "spec_helper"
describe Api::Upgrade do
let!(:prechecks) do
JSON.parse(
File.read(
"spec/fixtures/prechecks.json"
)
)
end
let!(:upgrade_status) do
JSON.parse(
File.read(
"spec/fixtures/upgrade_status.json"
)
)
end
let!(:node_repocheck) do
JSON.parse(
File.read(
"spec/fixtures/node_repocheck.json"
)
)
end
let!(:crowbar_repocheck) do
JSON.parse(
File.read(
"spec/fixtures/crowbar_repocheck.json"
)
)
end
let!(:crowbar_repocheck_zypper) do
File.read(
"spec/fixtures/crowbar_repocheck_zypper.xml"
).to_s
end
let!(:crowbar_repocheck_zypper_locked) do
File.read(
"spec/fixtures/crowbar_repocheck_zypper_locked.xml"
).to_s
end
let!(:crowbar_repocheck_zypper_prompt) do
File.read(
"spec/fixtures/crowbar_repocheck_zypper_prompt.xml"
).to_s
end
let(:pacemaker) do
Class.new
end
let(:barclamp_catalog) do
{
"nova" => {},
"database" => {},
"deployer" => {},
"pacemaker" => {},
"cinder" => {}
}
end
let(:database_proposal) { Proposal.create(barclamp: "database", name: "default") }
let(:cinder_proposal) { Proposal.create(barclamp: "cinder", name: "default") }
let(:nova_proposal) { Proposal.create(barclamp: "nova", name: "default") }
before(:each) do
allow(Api::Node).to(
receive(:node_architectures).and_return(
"os" => ["x86_64"],
"openstack" => ["x86_64"],
"ceph" => ["x86_64"],
"ha" => ["x86_64"]
)
)
allow(Node).to(
receive(:all).and_return([Node.find_by_name("testing.crowbar.com")])
)
allow(Api::Upgrade).to(
receive(:target_platform).and_return("suse-12.4")
)
allow(::Crowbar::Repository).to(
receive(:provided_and_enabled?).and_return(true)
)
["os", "ceph", "ha", "openstack"].each do |feature|
allow(::Crowbar::Repository).to(
receive(:provided_and_enabled_with_repolist_for_upgrade).with(
feature, "suse-12.4", "x86_64"
).and_return([true, {}])
)
end
stub_const("Api::Pacemaker", pacemaker)
allow(pacemaker).to receive(
:ha_presence_check
).and_return({})
allow(pacemaker).to receive(
:health_report
).and_return({})
end
context "with a successful status" do
it "checks the status" do
allow(Api::Upgrade).to receive(:network_checks).and_return([])
allow(Api::Upgrade).to receive(
:maintenance_updates_status
).and_return({})
allow(Api::Crowbar).to receive(
:addons
).and_return(["ceph", "ha"])
expect(subject.class).to respond_to(:status)
expect(subject.class.status).to be_a(Hash)
expect(subject.class.status.to_json).to eq(upgrade_status.to_json)
end
it "checks the node upgrade status" do
allow(Node).to receive(:all).and_return([Node.find_by_name("testing.crowbar.com")])
allow_any_instance_of(Node).to receive(:upgraded?).and_return(true)
allow_any_instance_of(Crowbar::UpgradeStatus).to receive(:passed?).with(:services).and_return(
true
)
expect(subject.class.node_status).to eq(
upgraded: ["testing.crowbar.com"],
not_upgraded: []
)
end
end
context "with a successful maintenance updates check" do
it "checks the maintenance updates on crowbar" do
allow(Crowbar::Sanity).to receive(:check).and_return([])
allow(Crowbar::Checks::Maintenance).to receive(
:updates_status
).and_return({})
allow(Api::Crowbar).to receive(
:addons
).and_return(["ceph", "ha"])
allow(Api::Crowbar).to receive(
:deployment_check
).and_return({})
allow(Api::Crowbar).to receive(
:health_check
).and_return({})
allow(Api::Crowbar).to receive(
:ha_config_check
).and_return({})
allow(Api::Crowbar).to receive(
:compute_status
).and_return({})
allow_any_instance_of(Crowbar::UpgradeStatus).to receive(
:start_step
).with(:prechecks).and_return(true)
allow_any_instance_of(Crowbar::UpgradeStatus).to receive(
:end_step
).and_return(true)
expect(subject.class.checks[:checks][:maintenance_updates_installed][:passed]).to be true
end
end
context "with a successful services shutdown" do
it "prepares and shuts down services on cluster founder nodes" do
allow_any_instance_of(Crowbar::UpgradeStatus).to receive(
:start_step
).with(:services).and_return(true)
allow_any_instance_of(CrowbarService).to receive(
:prepare_nodes_for_os_upgrade
).and_return(true)
allow(Api::Upgrade).to(
receive(:check_schema_migrations).and_return(true)
)
allow(Api::Upgrade).to(
receive(:check_product_version).and_return(true)
)
allow(Node).to(
receive(:find).with("state:crowbar_upgrade").and_return(
[Node.find_by_name("testing.crowbar.com")]
)
)
allow_any_instance_of(Node).to(
receive(:set_pre_upgrade_attribute).and_return([200, ""])
)
allow_any_instance_of(Node).to(
receive(:wait_for_script_to_finish).and_return(nil)
)
allow(Api::Upgrade).to receive(:execute_scripts_and_wait_for_finish).and_return(true)
allow_any_instance_of(Crowbar::UpgradeStatus).to receive(
:end_step
).and_return(true)
expect(subject.class.services_without_delay).to be true
end
end
context "with a failure during services shutdown" do
it "fails when chef client does not preapre the scripts" do
allow_any_instance_of(CrowbarService).to receive(
:prepare_nodes_for_os_upgrade
).and_raise("some Error")
allow_any_instance_of(Crowbar::UpgradeStatus).to receive(:start_step).and_return(true)
allow_any_instance_of(Crowbar::UpgradeStatus).to receive(:end_step).and_return(true)
expect(subject.class.services_without_delay).to be nil
end
end
context "with a successful node repocheck" do
it "checks the repositories for the nodes" do
os_repo_fixture = node_repocheck.tap do |k|
k.delete("ceph")
k.delete("ha")
end
allow_any_instance_of(Crowbar::UpgradeStatus).to receive(:start_step).and_return(true)
allow_any_instance_of(Crowbar::UpgradeStatus).to receive(:end_step).and_return(true)
allow_any_instance_of(ProvisionerService).to receive(:enable_repository).and_return(true)
expect(subject.class.noderepocheck).to eq(os_repo_fixture)
end
end
context "with addon installed but not deployed" do
it "shows that there are no addons deployed" do
allow_any_instance_of(Api::Crowbar).to(
receive(:features).and_return(
["ceph", "ha"]
)
)
allow(Api::Node).to(
receive(:ceph_node?).with(anything).and_return(false)
)
allow(Api::Node).to(
receive(:pacemaker_node?).with(anything).and_return(false)
)
allow(Api::Node).to(
receive(:node_architectures).and_return(
"os" => ["x86_64"],
"openstack" => ["x86_64"]
)
)
allow_any_instance_of(Crowbar::UpgradeStatus).to receive(:start_step).and_return(true)
allow_any_instance_of(Crowbar::UpgradeStatus).to receive(:end_step).and_return(true)
allow_any_instance_of(ProvisionerService).to receive(:enable_repository).and_return(true)
expected = node_repocheck.tap do |k|
k.delete("ceph")
k.delete("ha")
end
expect(subject.class.noderepocheck).to eq(expected)
end
end
context "with repositories not in place" do
it "lists the repositories that are not available" do
allow(Api::Upgrade).to(
receive(:repo_version_available?).and_return(false)
)
allow(Api::Upgrade).to(
receive(:admin_architecture).and_return("x86_64")
)
allow_any_instance_of(Kernel).to(
receive(:`).with(
"sudo /usr/bin/zypper-retry --xmlout products"
).and_return(crowbar_repocheck_zypper)
)
allow_any_instance_of(Crowbar::UpgradeStatus).to receive(
:start_step
).with(:repocheck_crowbar).and_return(true)
allow_any_instance_of(Crowbar::UpgradeStatus).to receive(
:end_step
).and_return(true)
expect(subject.class.adminrepocheck.deep_stringify_keys).to_not(
eq(crowbar_repocheck)
)
end
it "has only one repository that is not available" do
allow(Api::Upgrade).to(
receive(:repo_version_available?).with(
Hash.from_xml(crowbar_repocheck_zypper)["stream"]["product_list"]["product"],
"SLES",
"12.4"
).and_return(false)
)
allow(Api::Upgrade).to(
receive(:repo_version_available?).with(
Hash.from_xml(crowbar_repocheck_zypper)["stream"]["product_list"]["product"],
"suse-openstack-cloud-crowbar",
"8"
).and_return(true)
)
allow(Api::Upgrade).to(
receive(:admin_architecture).and_return("x86_64")
)
allow_any_instance_of(Kernel).to(
receive(:`).with(
"sudo /usr/bin/zypper-retry --xmlout products"
).and_return(crowbar_repocheck_zypper)
)
allow_any_instance_of(Crowbar::UpgradeStatus).to receive(
:start_step
).with(:repocheck_crowbar).and_return(true)
allow_any_instance_of(Crowbar::UpgradeStatus).to receive(
:end_step
).and_return(true)
expect(subject.class.adminrepocheck.deep_stringify_keys).to_not(
eq(crowbar_repocheck.merge(subject.class.adminrepocheck[:os]))
)
end
end
context "with a locked zypper" do
it "shows an error message that zypper is locked" do
allow(Api::Crowbar).to(
receive(:repo_version_available?).and_return(false)
)
allow_any_instance_of(Kernel).to(
receive(:`).with(
"sudo /usr/bin/zypper-retry --xmlout products"
).and_return(crowbar_repocheck_zypper_locked)
)
allow_any_instance_of(Crowbar::UpgradeStatus).to receive(
:start_step
).with(:repocheck_crowbar).and_return(true)
allow_any_instance_of(Crowbar::UpgradeStatus).to receive(
:end_step
).and_return(true)
check = subject.class.adminrepocheck
expect(check[:status]).to eq(:service_unavailable)
expect(check[:error]).to eq(
Hash.from_xml(crowbar_repocheck_zypper_locked)["stream"]["message"]
)
end
end
context "with a zypper prompt" do
it "shows the prompt text" do
allow(Api::Crowbar).to(
receive(:repo_version_available?).and_return(false)
)
allow_any_instance_of(Kernel).to(
receive(:`).with(
"sudo /usr/bin/zypper-retry --xmlout products"
).and_return(crowbar_repocheck_zypper_prompt)
)
allow_any_instance_of(Crowbar::UpgradeStatus).to receive(
:start_step
).with(:repocheck_crowbar).and_return(true)
allow_any_instance_of(Crowbar::UpgradeStatus).to receive(
:end_step
).and_return(true)
check = subject.class.adminrepocheck
expect(check[:status]).to eq(:service_unavailable)
expect(check[:error]).to eq(
"Zypper requires user interaction.\n" \
"\nMessage: " +
Hash.from_xml(crowbar_repocheck_zypper_prompt)["stream"]["message"] +
"\nPrompt: " +
Hash.from_xml(crowbar_repocheck_zypper_prompt)["stream"]["prompt"]["text"]
)
end
end
context "with repositories in place" do
it "lists the available repositories" do
allow(Api::Upgrade).to(
receive(:repo_version_available?).and_return(true)
)
allow_any_instance_of(Kernel).to(
receive(:`).with(
"sudo /usr/bin/zypper-retry --xmlout products"
).and_return(crowbar_repocheck_zypper)
)
allow_any_instance_of(Crowbar::UpgradeStatus).to receive(
:start_step
).with(:repocheck_crowbar).and_return(true)
allow_any_instance_of(Crowbar::UpgradeStatus).to receive(
:end_step
).and_return(true)
expect(subject.class.adminrepocheck.deep_stringify_keys).to(
eq(crowbar_repocheck)
)
end
end
context "while sorting elements for upgrade" do
before(:example) do
database_proposal.elements["database-server"] = ["data"]
cinder_proposal.elements["cinder-controller"] = ["services"]
cinder_proposal.elements["cinder-volume"] = ["services"]
nova_proposal.elements["nova-controller"] = ["services"]
nova_proposal.elements["nova-compute-kvm"] = ["compute"]
end
it "leaves node with nova-compute role" do
proposals = {
"database" => database_proposal,
"cinder" => cinder_proposal,
"nova" => nova_proposal
}
expect(subject.class.upgradable_elements_of_proposals(proposals)).to(
eq(["data", "services"])
)
end
it "takes node with cinder-volume when alone" do
cinder_proposal.elements["cinder-volume"] = ["storage"]
proposals = {
"database" => database_proposal,
"cinder" => cinder_proposal,
"nova" => nova_proposal
}
expect(subject.class.upgradable_elements_of_proposals(proposals)).to(
eq(["data", "services", "storage"])
)
end
it "leaves node with cinder-volume when on nova-compute" do
cinder_proposal.elements["cinder-volume"] = ["compute"]
proposals = {
"database" => database_proposal,
"cinder" => cinder_proposal,
"nova" => nova_proposal
}
expect(subject.class.upgradable_elements_of_proposals(proposals)).to(
eq(["data", "services"])
)
end
it "takes node with cinder-volume when alone and in cluster" do
cinder_proposal.elements["cinder-volume"] = ["cluster:compute"]
proposals = {
"database" => database_proposal,
"cinder" => cinder_proposal,
"nova" => nova_proposal
}
allow(ServiceObject).to receive(:expand_nodes_for_all).and_return(
[["storage"]]
)
expect(subject.class.upgradable_elements_of_proposals(proposals)).to(
eq(["data", "services", "storage"])
)
end
it "leaves node with cinder-volume when in cluster with nova-compute" do
cinder_proposal.elements["cinder-volume"] = ["cluster:compute"]
proposals = {
"database" => database_proposal,
"cinder" => cinder_proposal,
"nova" => nova_proposal
}
allow(ServiceObject).to receive(:expand_nodes_for_all).and_return(
[["compute"]]
)
expect(subject.class.upgradable_elements_of_proposals(proposals)).to(
eq(["data", "services"])
)
end
end
context "upgrading the nodes in normal mode" do
it "successfully upgrades controller nodes" do
allow(Api::Upgrade).to receive(:remaining_nodes).and_return(1)
allow(Api::Upgrade).to receive(:upgrade_mode).and_return(:normal)
allow(BarclampCatalog).to receive(:barclamps).and_return(barclamp_catalog)
allow(BarclampCatalog).to receive(:category).and_return("OpenStack")
allow(BarclampCatalog).to receive(:category).with("deployer").and_return("Crowbar")
cinder_proposal.elements["cinder-controller"] = ["drbd"]
nova_proposal.elements["nova-controller"] = ["ceph"]
nova_proposal.elements["nova-compute-kvm"] = ["testing"]
allow(Proposal).to receive(:where).with(barclamp: "database").and_return([database_proposal])
allow(Proposal).to receive(:where).with(barclamp: "cinder").and_return([cinder_proposal])
allow(Proposal).to receive(:where).with(barclamp: "nova").and_return([nova_proposal])
allow_any_instance_of(Proposal).to(receive(:active?).and_return(true))
allow(BarclampCatalog).to receive(:run_order).with("database").and_return(1)
allow(BarclampCatalog).to receive(:run_order).with("cinder").and_return(2)
allow(BarclampCatalog).to receive(:run_order).with("nova").and_return(3)
allow(Api::Upgrade).to receive(:upgrade_one_node).and_return(true)
# rest of the upgrade, after controller nodes
allow(Api::Upgrade).to receive(:prepare_all_compute_nodes).and_return(true)
allow(Api::Upgrade).to receive(:upgrade_all_compute_nodes).and_return(true)
allow(Api::Upgrade).to receive(:finalize_nodes_upgrade).and_return(true)
allow_any_instance_of(Crowbar::UpgradeStatus).to receive(
:start_step
).with(:nodes).and_return(true)
allow_any_instance_of(Crowbar::UpgradeStatus).to receive(
:progress
).and_return(remaining_nodes: 0)
allow_any_instance_of(Crowbar::UpgradeStatus).to receive(:end_step).and_return(true)
expect(subject.class.nodes_without_delay).to be true
end
it "successfully upgrades compute nodes" do
node1 = Node.find_by_name("testing.crowbar.com")
node2 = Node.find_by_name("ceph.crowbar.com")
allow(Api::Upgrade).to receive(:remaining_nodes).and_return(2)
allow(Api::Upgrade).to receive(:upgrade_mode).and_return(:normal)
allow(Api::Upgrade).to receive(:do_controllers_substep).and_return(true)
allow(Node).to(
receive(:find).with("roles:nova-compute-*").and_return([node1, node2])
)
# parallel_upgrade_compute_nodes:
allow(Api::Upgrade).to receive(:execute_scripts_and_wait_for_finish).with(
[node1, node2],
"/usr/sbin/crowbar-upgrade-os.sh",
::Crowbar::UpgradeTimeouts.new.values[:upgrade_os]
).and_return(true)
allow_any_instance_of(Node).to receive(:upgraded?).and_return(false)
allow_any_instance_of(Node).to receive(:ready_after_upgrade?).and_return(false)
allow_any_instance_of(Api::Node).to receive(:save_node_state).and_return(true)
allow_any_instance_of(Api::Node).to receive(:reboot_and_wait).and_return(true)
allow_any_instance_of(Api::Node).to receive(:join_and_chef).and_return(true)
allow(Api::Upgrade).to receive(:finalize_nodes_upgrade).and_return(true)
allow_any_instance_of(Crowbar::UpgradeStatus).to receive(
:start_step
).with(:nodes).and_return(true)
allow_any_instance_of(Crowbar::UpgradeStatus).to receive(
:progress
).and_return(remaining_nodes: 0)
allow_any_instance_of(Crowbar::UpgradeStatus).to receive(:end_step).and_return(true)
expect(subject.class.nodes_without_delay).to be true
end
end
context "upgrading the nodes in non-disruptive mode" do
it "successfully upgrades the storage nodes" do
ceph = Node.find_by_name("ceph.crowbar.com")
testing = Node.find_by_name("testing.crowbar.com")
allow(Node).to(receive(:find).with("state:crowbar_upgrade").and_return([testing]))
allow(Api::Upgrade).to receive(:upgrade_controller_clusters).and_return(true)
# upgrade_non_compute_nodes:
allow(Node).to(
receive(:find).with(
"NOT run_list_map:nova-compute-*"
).and_return([ceph, testing])
)
allow_any_instance_of(Api::Node).to receive(:save_node_state).with(
"controller", "upgrading"
).and_return(true)
allow_any_instance_of(Api::Node).to receive(:upgrade).and_return(true)
allow_any_instance_of(Api::Node).to receive(:reboot_and_wait).and_return(true)
allow_any_instance_of(Api::Node).to receive(:post_upgrade).and_return(true)
allow_any_instance_of(Api::Node).to receive(:join_and_chef).and_return(true)
allow_any_instance_of(Api::Node).to receive(:save_node_state).with(
"controller", "upgraded"
).and_return(true)
allow(Api::Upgrade).to receive(:prepare_all_compute_nodes).and_return(true)
allow(Api::Upgrade).to receive(:upgrade_all_compute_nodes).and_return(true)
allow_any_instance_of(Crowbar::UpgradeStatus).to receive(
:progress
).and_return(remaining_nodes: 0)
allow(Api::Upgrade).to receive(:stop_nova_services).and_return(true)
allow(Api::Upgrade).to receive(:reload_nova_services).and_return(true)
allow(Api::Upgrade).to receive(:run_online_migrations).and_return(true)
allow(Api::Upgrade).to receive(:finalize_nodes_upgrade).and_return(true)
allow_any_instance_of(Crowbar::UpgradeStatus).to receive(:end_step).and_return(true)
expect(subject.class.nodes_without_delay).to be true
end
it "successfully completes the upgrade when they are no compute nodes" do
allow(Api::Upgrade).to receive(:remaining_nodes).and_return(1)
allow(Api::Upgrade).to receive(:do_controllers_substep).and_return(true)
allow(Node).to(
receive(:find).with("roles:nova-compute-kvm").and_return([])
)
allow_any_instance_of(Crowbar::UpgradeStatus).to receive(
:start_step
).with(:nodes).and_return(true)
allow_any_instance_of(Crowbar::UpgradeStatus).to receive(
:progress
).and_return(remaining_nodes: 0)
allow(Api::Upgrade).to receive(:stop_nova_services).and_return(true)
allow(Api::Upgrade).to receive(:reload_nova_services).and_return(true)
allow(Api::Upgrade).to receive(:run_online_migrations).and_return(true)
allow(Api::Upgrade).to receive(:finalize_nodes_upgrade).and_return(true)
allow_any_instance_of(Crowbar::UpgradeStatus).to receive(:end_step).and_return(true)
expect(subject.class.nodes_without_delay).to be true
end
it "fails with some non-upgrade error" do
allow_any_instance_of(Crowbar::UpgradeStatus).to receive(
:start_step
).with(:nodes).and_return(true)
allow(Node).to(receive(:find).and_raise("Some Error"))
allow_any_instance_of(Crowbar::UpgradeStatus).to(
receive(:end_step).and_return(false)
)
expect { subject.class.nodes_without_delay }.to raise_error(RuntimeError)
end
it "fails to upgrade compute nodes when there is no nova-controller" do
allow_any_instance_of(Crowbar::UpgradeStatus).to receive(
:start_step
).with(:nodes).and_return(true)
allow(Api::Upgrade).to receive(:remaining_nodes).and_return(1)
allow(Api::Upgrade).to receive(:do_controllers_substep).and_return(true)
allow(Node).to(
receive(:find).with("roles:nova-compute-kvm").
and_return([Node.find_by_name("testing.crowbar.com")])
)
allow_any_instance_of(Node).to receive(:upgraded?).and_return(false)
allow(Node).to(
receive(:find).with("roles:nova-controller").and_return([])
)
allow_any_instance_of(Crowbar::UpgradeStatus).to(
receive(:end_step).
with(
false,
nodes: {
data:
"No node with 'nova-controller' role node was found. " \
"Cannot proceed with upgrade of compute nodes.",
help:
"Check the log files at the node that has failed " \
"to find possible cause."
}
).
and_return(false)
)
expect(subject.class.nodes_without_delay).to be false
end
it "during the upgrade of controller nodes, detect that they are upgraded" do
node1 = Node.find_by_name("testing.crowbar.com")
allow_any_instance_of(Crowbar::UpgradeStatus).to receive(
:start_step
).with(:nodes).and_return(true)
allow(Api::Upgrade).to receive(:remaining_nodes).and_return(1)
allow(Node).to(
receive(:find).
with(
"run_list_map:pacemaker-cluster-member AND run_list_map:neutron-network " \
"AND NOT run_list_map:neutron-server"
).and_return([])
)
allow(Node).to(
receive(:find).with(
"run_list_map:pacemaker-cluster-member"
).and_return([node1])
)
allow(Node).to(
receive(:find).with("testing.crowbar.com").and_return([])
)
allow(Node).to(
receive(:find).with(
"pacemaker_config_environment:data " \
"AND run_list_map:pacemaker-cluster-member " \
"AND NOT fqdn:testing.crowbar.com"
).and_return([])
)
allow_any_instance_of(Node).to receive(:upgraded?).and_return(true)
allow(Api::Upgrade).to receive(:upgrade_non_compute_nodes).and_return(true)
allow(Api::Upgrade).to receive(:prepare_all_compute_nodes).and_return(true)
allow(Api::Upgrade).to receive(:upgrade_all_compute_nodes).and_return(true)
allow_any_instance_of(Crowbar::UpgradeStatus).to receive(
:progress
).and_return(remaining_nodes: 0)
# mock the node seach for stop_nova_services
allow(Node).to(
receive(:find).with("roles:nova-*").and_return([node1])
)
allow_any_instance_of(Node).to(
receive(
:wait_for_script_to_finish
).with(
"/usr/sbin/crowbar-stop-nova-services.sh", 120
).and_return(nil)
)
allow(Api::Upgrade).to receive(:reload_nova_services).and_return(true)
allow(Api::Upgrade).to receive(:run_online_migrations).and_return(true)
allow(Api::Upgrade).to receive(:finalize_nodes_upgrade).and_return(true)
allow_any_instance_of(Crowbar::UpgradeStatus).to receive(:end_step).and_return(true)
expect(subject.class.nodes_without_delay).to be true
end
it "detects that compute nodes are already upgraded during preparation" do
allow_any_instance_of(Crowbar::UpgradeStatus).to receive(
:start_step
).with(:nodes).and_return(true)
allow(Api::Upgrade).to receive(:remaining_nodes).and_return(1)
allow(Api::Upgrade).to receive(:do_controllers_substep).and_return(true)
allow(Node).to(
receive(:find).with("roles:nova-compute-kvm").
and_return([Node.find_by_name("testing.crowbar.com")])
)
allow_any_instance_of(Node).to receive(:upgraded?).and_return(true)
allow(Api::Upgrade).to receive(:upgrade_all_compute_nodes).and_return(true)
allow_any_instance_of(Crowbar::UpgradeStatus).to receive(
:progress
).and_return(remaining_nodes: 0)
allow(Api::Upgrade).to receive(:reload_nova_services).and_return(true)
allow(Api::Upgrade).to receive(:run_online_migrations).and_return(true)
allow(Api::Upgrade).to receive(:finalize_nodes_upgrade).and_return(true)
allow_any_instance_of(Crowbar::UpgradeStatus).to receive(:end_step).and_return(true)
expect(subject.class.nodes_without_delay).to be true
end
it "during the upgrade of compute nodes, detect that they are upgraded" do
allow_any_instance_of(Crowbar::UpgradeStatus).to receive(
:start_step
).with(:nodes).and_return(true)
allow(Api::Upgrade).to receive(:remaining_nodes).and_return(1)
allow(Api::Upgrade).to receive(:do_controllers_substep).and_return(true)
allow_any_instance_of(Crowbar::UpgradeStatus).to receive(:save_substep).and_return(true)
allow(Node).to(
receive(:find).with("roles:nova-compute-kvm").
and_return([Node.find_by_name("testing.crowbar.com")])
)
allow_any_instance_of(Node).to receive(:upgraded?).and_return(true)
allow_any_instance_of(Crowbar::UpgradeStatus).to receive(
:progress
).and_return(remaining_nodes: 0)
allow(Api::Upgrade).to receive(:reload_nova_services).and_return(true)
allow(Api::Upgrade).to receive(:run_online_migrations).and_return(true)
allow(Api::Upgrade).to receive(:finalize_nodes_upgrade).and_return(true)
allow_any_instance_of(Crowbar::UpgradeStatus).to receive(:end_step).and_return(true)
expect(subject.class.nodes_without_delay).to be true
end
it "successfully upgrades KVM compute nodes" do
allow(Api::Upgrade).to receive(:remaining_nodes).and_return(1)
allow(Api::Upgrade).to receive(:do_controllers_substep).and_return(true)
allow(Node).to(
receive(:find).with("roles:nova-compute-kvm").
and_return([Node.find_by_name("testing.crowbar.com")])
)
allow(Node).to(
receive(:find).with("roles:nova-controller").
and_return([Node.find_by_name("testing.crowbar.com")])
)
allow(Api::Upgrade).to receive(:execute_scripts_and_wait_for_finish).and_return(true)
allow_any_instance_of(Node).to receive(:upgraded?).and_return(false)
allow_any_instance_of(Api::Node).to receive(:save_node_state).and_return(true)
allow(Api::Upgrade).to receive(:live_evacuate_compute_node).and_return(true)
allow_any_instance_of(Api::Node).to receive(:os_upgrade).and_return(true)
allow_any_instance_of(Api::Node).to receive(:reboot_and_wait).and_return(true)
allow_any_instance_of(Api::Node).to receive(:post_upgrade).and_return(true)
allow_any_instance_of(Api::Node).to receive(:join_and_chef).and_return(true)
allow_any_instance_of(Node).to receive(:run_ssh_cmd).and_return(
exit_code: 0,
stdout: "enabled"
)
allow_any_instance_of(Crowbar::UpgradeStatus).to receive(
:start_step
).with(:nodes).and_return(true)
allow_any_instance_of(Crowbar::UpgradeStatus).to receive(
:progress
).and_return(remaining_nodes: 0)
allow(Api::Upgrade).to receive(:reload_nova_services).and_return(true)
allow(Api::Upgrade).to receive(:run_online_migrations).and_return(true)
allow(Api::Upgrade).to receive(:finalize_nodes_upgrade).and_return(true)
allow_any_instance_of(Crowbar::UpgradeStatus).to receive(:end_step).and_return(true)
expect(subject.class.nodes_without_delay).to be true
end
end
context "canceling the upgrade" do
it "successfully cancels the upgrade" do
allow_any_instance_of(ProvisionerService).to receive(
:enable_all_repositories
).and_return(true)
allow_any_instance_of(CrowbarService).to receive(
:revert_nodes_from_crowbar_upgrade
).and_return(true)
allow_any_instance_of(Crowbar::UpgradeStatus).to receive(
:initialize_state
).and_return(true)
allow_any_instance_of(Crowbar::UpgradeStatus).to receive(
:cancel_allowed?
).and_return(true)
expect(subject.class.cancel).to be true
end
it "fails to cancel the upgrade" do
allow_any_instance_of(CrowbarService).to receive(
:revert_nodes_from_crowbar_upgrade
).and_raise("Some Error")
allow_any_instance_of(Crowbar::UpgradeStatus).to receive(:current_step).and_return(:database)
expect { subject.class.cancel }.to raise_error(Crowbar::Error::Upgrade::CancelError)
end
it "is allowed to cancel the upgrade" do
allow_any_instance_of(ProvisionerService).to receive(
:enable_all_repositories
).and_return(true)
allow_any_instance_of(CrowbarService).to receive(
:revert_nodes_from_crowbar_upgrade
).and_return(true)
allow_any_instance_of(Crowbar::UpgradeStatus).to receive(
:save
).and_return(true)
allow_any_instance_of(Crowbar::UpgradeStatus).to receive(
:running?
).and_return(false)
[
:prechecks,
:prepare,
:backup_crowbar,
:repocheck_crowbar,
:admin
].each do |allowed_step|
allow_any_instance_of(Crowbar::UpgradeStatus).to receive(
:current_step
).and_return(allowed_step)
expect(subject.class.cancel).to be true
end
end
it "is not allowed to cancel the upgrade" do
allow_any_instance_of(CrowbarService).to receive(
:revert_nodes_from_crowbar_upgrade
).and_return(true)
allow_any_instance_of(Crowbar::UpgradeStatus).to receive(
:save
).and_return(true)
[
:admin,
:database,
:repocheck_nodes,
:services,
:backup_openstack,
:nodes,
:finished
].each do |allowed_step|
if allowed_step == :admin
allow_any_instance_of(Crowbar::UpgradeStatus).to receive(
:running?
).and_return(true)
end
allow_any_instance_of(Crowbar::UpgradeStatus).to receive(
:current_step
).and_return(allowed_step)
expect { subject.class.cancel }.to raise_error(Crowbar::Error::Upgrade::CancelError)
end
end
it "is not allowed to cancel the upgrade while crowbar is running" do
allow_any_instance_of(CrowbarService).to receive(
:revert_nodes_from_crowbar_upgrade
).and_return(true)
allow_any_instance_of(Crowbar::UpgradeStatus).to receive(
:current_step
).and_return(:admin)
allow_any_instance_of(Crowbar::UpgradeStatus).to receive(
:running?
).and_return(true)
expect { subject.class.cancel }.to raise_error(Crowbar::Error::Upgrade::CancelError)
end
end
context "determining the best upgrade method" do
it "chooses non-disruptive upgrade when all prechecks succeed" do
allow(Crowbar::Sanity).to receive(:check).and_return([])
allow(Crowbar::Checks::Maintenance).to receive(
:updates_status
).and_return({})
allow(Api::Crowbar).to receive(
:addons
).and_return(["ceph", "ha"])
allow(Api::Crowbar).to receive(
:deployment_check
).and_return({})
allow(Api::Pacemaker).to(
receive(:ha_presence_check).and_return({})
)
allow(Api::Crowbar).to(
receive(:health_check).and_return({})
)
allow(Api::Crowbar).to(
receive(:ceph_status).and_return({})
)
allow(Api::Crowbar).to receive(
:ha_config_check
).and_return({})
allow(Api::Crowbar).to(
receive(:compute_status).and_return({})
)
allow_any_instance_of(Crowbar::UpgradeStatus).to receive(:start_step).and_return(true)
allow_any_instance_of(Crowbar::UpgradeStatus).to receive(:end_step).and_return(true)
expect(subject.class.checks.deep_symbolize_keys[:best_method]).to be :non_disruptive
end
it "chooses 'normal' upgrade when a non-required prechecks fails" do
allow(Crowbar::Sanity).to receive(:check).and_return([])
allow(Crowbar::Checks::Maintenance).to receive(
:updates_status
).and_return({})
allow(Api::Crowbar).to receive(
:addons
).and_return(["ceph", "ha"])
allow(Api::Crowbar).to receive(
:deployment_check
).and_return({})
allow(Api::Pacemaker).to receive(
:ha_presence_check
).and_return(error: "ERROR")
allow(Api::Crowbar).to(
receive(:health_check).and_return({})
)
allow(Api::Crowbar).to(
receive(:ceph_status).and_return({})
)
allow(Api::Crowbar).to receive(
:ha_config_check
).and_return({})
allow(Api::Crowbar).to(
receive(:compute_status).and_return({})
)
allow_any_instance_of(Crowbar::UpgradeStatus).to receive(:start_step).and_return(true)
allow_any_instance_of(Crowbar::UpgradeStatus).to receive(:end_step).and_return(true)
expect(subject.class.checks.deep_symbolize_keys[:best_method]).to be :normal
end
it "chooses none when a required precheck fails" do
allow(Crowbar::Sanity).to receive(:check).and_return([])
allow(Crowbar::Checks::Maintenance).to receive(
:updates_status
).and_return(errors: ["Some Error"])
allow(Api::Crowbar).to receive(
:ha_config_check
).and_return({})
allow(Api::Crowbar).to receive(
:deployment_check
).and_return({})
allow(Api::Pacemaker).to receive(
:health_report
).and_return(crm_failures: "error", failed_actions: "error")
allow(Api::Crowbar).to receive(:compute_status).and_return({})
allow_any_instance_of(Crowbar::UpgradeStatus).to receive(:start_step).and_return(true)
allow_any_instance_of(Crowbar::UpgradeStatus).to receive(:end_step).and_return(true)
expect(subject.class.checks[:best_method]).to be :none
end
end
context "setting the upgrade mode" do
it "returns an error when an invalid upgrade mode is set" do
expect { subject.class.upgrade_mode = "invalid" }.to raise_error(
Crowbar::Error::SaveUpgradeModeError
)
end
end
context "with preparing the upgrade" do
it "succeeds to spawn the prepare in the background" do
allow_any_instance_of(CrowbarService).to receive(
:prepare_nodes_for_crowbar_upgrade
).and_return(true)
allow_any_instance_of(Crowbar::UpgradeStatus).to receive(
:start_step
).with(:prepare).and_return(true)
expect(subject.class.prepare(background: true)).to be true
end
it "succeeds to spawn the prepare in the foreground" do
allow_any_instance_of(CrowbarService).to receive(
:prepare_nodes_for_crowbar_upgrade
).and_return(true)
allow_any_instance_of(Crowbar::UpgradeStatus).to receive(
:start_step
).with(:prepare).and_return(true)
allow_any_instance_of(Crowbar::UpgradeStatus).to receive(
:end_step
).and_return(true)
expect(subject.class.prepare).to be true
end
it "fails to spawn the prepare in the foreground" do
allow_any_instance_of(CrowbarService).to receive(
:prepare_nodes_for_crowbar_upgrade
).and_raise("Some error")
allow_any_instance_of(Crowbar::UpgradeStatus).to receive(
:start_step
).with(:prepare).and_return(true)
allow_any_instance_of(Crowbar::UpgradeStatus).to receive(
:end_step
).and_return(true)
expect(subject.class.prepare).to be false
end
end
context "with a successful backup creation for OpenStack" do
let(:size_query) { "SELECT SUM(data_length + index_length) FROM information_schema.tables ;" }
let(:size_cmd) { "echo \"#{size_query}\" | mysql -N -u root -psecret" }
let(:db_node) do
Node.find_by_name("testing.crowbar.com")
end
it "creates a backup for OpenStack" do
allow_any_instance_of(Crowbar::UpgradeStatus).to receive(
:start_step
).with(:backup_openstack).and_return(true)
allow(::Node).to receive(:find).with("roles:database-config-default").and_return(
[db_node]
)
allow(db_node).to receive(:run_ssh_cmd).with(
size_cmd,
"60s"
).and_return(
exit_code: 0,
stdout: "12345678"
)
allow(File).to receive(:exist?).with(
"/var/lib/crowbar/backup/8-to-9-openstack_dump.sql.gz"
).and_return(false)
allow(Api::Upgrade).to receive(:run_cmd).and_return(
exit_code: 0,
stdout_and_stderr: ""
)
allow_any_instance_of(Crowbar::UpgradeStatus).to receive(
:end_step
).and_return(true)
expect(subject.class.openstackbackup_without_delay).to be true
end
it "finds out that an OpenStack backup has already been created" do
allow_any_instance_of(Crowbar::UpgradeStatus).to receive(
:start_step
).with(:backup_openstack).and_return(true)
allow(File).to receive(:exist?).with(
"/var/lib/crowbar/backup/8-to-9-openstack_dump.sql.gz"
).and_return(true)
allow_any_instance_of(Crowbar::UpgradeStatus).to receive(
:end_step
).and_return(true)
expect(subject.class.openstackbackup_without_delay).to be nil
end
end
context "with a failed backup creation for OpenStack" do
let(:crowbar_lib_dir) { "/var/lib/crowbar" }
let(:dump_path) { "#{crowbar_lib_dir}/backup/8-to-9-openstack_dump.sql.gz" }
let(:size_query) { "SELECT SUM(data_length + index_length) FROM information_schema.tables ;" }
let(:size_cmd) { "echo \"#{size_query}\" | mysql -N -u root -psecret" }
let(:dump_cmd) do
"sudo ssh -o ConnectTimeout=10 root@testing.crowbar.com " \
"\'mysqldump -u root -psecret --all-databases | gzip\' " \
"> #{dump_path}"
end
let(:disk_space_cmd) do
"LANG=C df -x 'tmpfs' -x 'devtmpfs' -B1 -l --output='avail' #{crowbar_lib_dir} | tail -n1"
end
let(:db_node) do
Node.find_by_name("testing.crowbar.com")
end
it "fails to create a backup for OpenStack" do
allow_any_instance_of(Crowbar::UpgradeStatus).to receive(
:start_step
).with(:backup_openstack).and_return(true)
allow(::Node).to receive(:find).with("roles:database-config-default").and_return(
[db_node]
)
allow(File).to receive(:exist?).with(dump_path).and_return(false)
allow(db_node).to receive(:run_ssh_cmd).with(
size_cmd,
"60s"
).and_return(
exit_code: 0,
stdout: "12345678"
)
allow(Api::Upgrade).to receive(:run_cmd).with(
disk_space_cmd
).and_return(
exit_code: 0,
stdout_and_stderr: ""
)
allow(Api::Upgrade).to receive(:run_cmd).with(
dump_cmd
).and_return(
exit_code: 1,
stdout_and_stderr: "Error"
)
allow_any_instance_of(Crowbar::UpgradeStatus).to receive(
:end_step
).and_return("rescued and set status to failed")
expect(subject.class.openstackbackup_without_delay).to eq "rescued and set status to failed"
end
it "fails to determine the accumulated size of the OpenStack databases" do
allow_any_instance_of(Crowbar::UpgradeStatus).to receive(
:start_step
).with(:backup_openstack).and_return(true)
allow(::Node).to receive(:find).with("roles:database-config-default").and_return(
[db_node]
)
allow(File).to receive(:exist?).with(dump_path).and_return(false)
allow(db_node).to receive(:run_ssh_cmd).with(
size_cmd,
"60s"
).and_return(
exit_code: 1,
stdout: "12345678",
stderr: "Error"
)
allow_any_instance_of(Crowbar::UpgradeStatus).to receive(
:end_step
).and_return("rescued and set status to failed")
expect(subject.class.openstackbackup_without_delay).to eq "rescued and set status to failed"
end
it "fails to determine the free disk space on the system" do
allow_any_instance_of(Crowbar::UpgradeStatus).to receive(
:start_step
).with(:backup_openstack).and_return(true)
allow(::Node).to receive(:find).with("roles:database-config-default").and_return(
[db_node]
)
allow(File).to receive(:exist?).with(dump_path).and_return(false)
allow(Api::Upgrade).to receive(:run_cmd).with(disk_space_cmd).and_return(
exit_code: 1,
stdout_and_stderr: "Error"
)
allow(db_node).to receive(:run_ssh_cmd).with(
size_cmd,
"60s"
).and_return(
exit_code: 0,
stdout: "12345678"
)
allow(Api::Upgrade).to receive(:run_cmd).with(
dump_cmd
).and_return(
exit_code: 0,
stdout_and_stderr: ""
)
allow_any_instance_of(Crowbar::UpgradeStatus).to receive(
:end_step
).and_return("rescued and set status to failed")
expect(subject.class.openstackbackup_without_delay).to eq "rescued and set status to failed"
end
it "fails to create the OpenStack backup due to not enough disk space available" do
allow_any_instance_of(Crowbar::UpgradeStatus).to receive(
:start_step
).with(:backup_openstack).and_return(true)
allow(::Node).to receive(:find).with("roles:database-config-default").and_return(
[db_node]
)
allow(File).to receive(:exist?).with(dump_path).and_return(false)
allow(db_node).to receive(:run_ssh_cmd).with(
size_cmd,
"60s"
).and_return(
exit_code: 0,
stdout: "1000000"
)
allow(Api::Upgrade).to receive(:run_cmd).with(disk_space_cmd).and_return(
exit_code: 0,
stdout_and_stderr: "999999"
)
allow_any_instance_of(Crowbar::UpgradeStatus).to receive(
:end_step
).and_return("rescued and set status to failed")
expect(subject.class.openstackbackup_without_delay).to eq "rescued and set status to failed"
end
end
end