modules/mu/deploy.rb
# Copyright:: Copyright (c) 2014 eGlobalTech, Inc., all rights reserved
#
# Licensed under the BSD-3 license (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License in the root of the project or at
#
# http://egt-labs.com/mu/LICENSE.html
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
require "net/http"
require "net/smtp"
require 'json'
require 'rexml/document'
module MU
# The Deploy class is the main interface for resource creation. It is
# typically invoked from the *mu-deploy* utility. It consumes a configuration
# parsed by {MU::Config} and generates cloud artifacts accordingly, ordering
# them per their dependencies and handing off to OS management tools (e.g.
# Chef) for application-level orchestration.
class Deploy
Thread.current.thread_variable_set("name", "main_thread");
# These also exist as instance variables, but we end up needing versions of
# them in static contexts too.
@deploy_semaphore = Mutex.new
# The name of the application which we're building.
attr_reader :appname
# The timestamp at which this deployment was begun
attr_reader :timestamp
# The environment into which we're deploying
attr_reader :environment
# The cloud provider's account identifier
attr_reader :account_number
# This flag indicates that cleanup operations should be skipped if a
# failure occurs.
attr_reader :nocleanup
# We just pass this flag to MommaCat, telling it not to save any metadata.
attr_reader :no_artifacts
# The deployment object we create for our stack
attr_reader :mommacat
# Indicates whether we are updating an existing deployment, as opposed to
# creating a new one.
attr_reader :updating
# @param environment [String]: The environment name for this application stack (e.g. "dev" or "prod")
# @param verbosity [Integer]: Debug level for MU.log output
# @param webify_logs [Boolean]: Toggles web-friendly log output
# @param nocleanup [Boolean]: Toggles whether to skip cleanup of resources if this deployment fails.
# @param cloudformation_path [String]: If we're outputting CloudFormation, here's where to put it
# @param force_cloudformation [Boolean]: Output CloudFormation regardless of what cloud resources target
# @param reraise_thread [Thread]: Raise any major exceptions to this thread
# @param stack_conf [Hash]: A full application stack configuration parsed by {MU::Config}
# @param no_artifacts [Boolean]: Do not save deploy metadata
# @param deploy_id [String]: Reload and re-process an existing deploy
def initialize(environment,
verbosity: MU::Logger::NORMAL,
color: true,
webify_logs: false,
nocleanup: false,
cloudformation_path: nil,
force_cloudformation: false,
reraise_thread: nil,
stack_conf: nil,
no_artifacts: false,
deploy_id: nil,
deploy_obj: nil)
MU.setVar("verbosity", verbosity)
MU.setVar("color", color)
@webify_logs = webify_logs
@verbosity = verbosity
@color = color
@nocleanup = nocleanup
@no_artifacts = no_artifacts
@reraise_thread = reraise_thread
MU.setLogging(verbosity, webify_logs, STDOUT, color)
MU::Cloud::CloudFormation.emitCloudFormation(set: force_cloudformation)
@cloudformation_output = cloudformation_path
if stack_conf.nil? or !stack_conf.is_a?(Hash)
raise MuError, "Deploy objects require a stack_conf hash"
end
@my_threads = Array.new
@last_sigterm = 0
@dependency_threads = {}
@dependency_semaphore = Mutex.new
@main_config = stack_conf
@original_config = Marshal.load(Marshal.dump(MU.structToHash(stack_conf.dup)))
@original_config.freeze
@admins = stack_conf["admins"]
@mommacat = deploy_obj
if deploy_id
@mommacat ||= MU::MommaCat.new(deploy_id)
@updating = true
else
@environment = environment
@updating = false
time=Time.new
@appname = stack_conf["appname"]
@timestamp = time.strftime("%Y%m%d%H").to_s
@timestamp.freeze
@timestart = time.to_s;
@timestart.freeze
retries = 0
begin
raise MuError, "Failed to allocate an unused MU-ID after #{retries} tries!" if retries > 70
seedsize = 1 + (retries/10).abs
seed = (0...seedsize+1).map { ('a'..'z').to_a[rand(26)] }.join
deploy_id = @appname.upcase + "-" + @environment.upcase + "-" + @timestamp + "-" + seed.upcase
end while MU::MommaCat.deploy_exists?(deploy_id) or seed == "mu"
MU.setVar("deploy_id", deploy_id)
MU.setVar("appname", @appname.upcase)
MU.setVar("environment", @environment.upcase)
MU.setVar("timestamp", @timestamp)
MU.setVar("mommacat", @mommacat)
MU.setVar("seed", seed)
MU.setVar("handle", MU::MommaCat.generateHandle(seed))
MU.log "Deployment id: #{MU.appname} \"#{MU.handle}\" (#{MU.deploy_id})"
end
@fromName = MU.muCfg['mu_admin_email']
MU::Cloud.resource_types.values.each { |data|
if !@main_config[data[:cfg_plural]].nil? and @main_config[data[:cfg_plural]].size > 0
@main_config[data[:cfg_plural]].each { |resource|
if force_cloudformation
if resource['cloud'] == "AWS"
resource['cloud'] = "CloudFormation"
if resource.has_key?("vpc") and resource["vpc"].is_a?(Hash)
resource["vpc"]['cloud'] = "CloudFormation"
elsif resource.has_key?("vpcs") and resource["vpcs"].is_a?(Array)
resource['vpcs'].each { |v| v['cloud'] = "CloudFormation" }
end
end
end
}
_shortclass, _cfg_name, _cfg_plural, classname = MU::Cloud.getResourceNames(data[:cfg_plural])
@main_config[data[:cfg_plural]].each { |resource|
resource["#MU_CLOUDCLASS"] = classname
# resource["#MU_CLOUDCLASS"] = MU::Cloud.resourceClass(resource['cloud'], data[:cfg_plural])
}
setThreadDependencies(@main_config[data[:cfg_plural]])
end
}
end
# Activate this deployment, instantiating all resources, orchestrating them,
# and saving metadata about them.
def run
Signal.trap("INT") do
# Don't use MU.log in here, it does a synchronize {} and that ain't
# legal inside a trap.
die = true if (Time.now.to_i - @last_sigterm) < 5
if !die and !MU::MommaCat.trapSafeLocks.nil? and MU::MommaCat.trapSafeLocks.size > 0
puts "------------------------------"
puts "Thread and lock debugging data"
puts "------------------------------"
puts "Open flock() locks:"
pp MU::MommaCat.trapSafeLocks
puts "------------------------------"
end
Thread.list.each do |t|
next if !t.status # skip threads that've been cleanly terminated
if !die
thread_name = t.thread_variable_get("name")
puts "Thread #{thread_name} (#{t.object_id}): #{t.inspect} #{t.status}"
t.thread_variables.each { |tvar|
puts "#{tvar} = #{t.thread_variable_get(tvar)}"
}
pp t.backtrace
if !@dependency_threads[thread_name].nil?
puts ""
puts "Waiting on #{@dependency_threads[thread_name]}"
Thread.list.each { |parent|
parent_name = parent.thread_variable_get("name")
if @dependency_threads[thread_name].include?(parent_name)
puts "\t#{parent_name} (#{parent.object_id}): #{parent.inspect} #{parent.status}"
parent.thread_variables.each { |tvar|
puts "\t#{tvar} = #{parent.thread_variable_get(tvar)}"
}
end
}
end
puts "------------------------------"
t.run
end
end
if !die
puts "Received SIGINT, hit ctrl-C again within five seconds to kill this deployment."
else
Thread.list.each do |t|
next if !t.status
if t.object_id != Thread.current.object_id and
t.thread_variable_get("name") != "main_thread" and
t.thread_variable_get("owned_by_mu")
t.kill
end
end
if @main_thread
@main_thread.raise "Terminated by user"
else
raise "Terminated by user"
end
end
@last_sigterm = Time.now.to_i
end
begin
@main_thread = Thread.current
if !@mommacat
metadata = {
"appname" => @appname,
"timestamp" => @timestamp,
"environment" => @environment,
"seed" => MU.seed,
"deployment_start_time" => @timestart,
"chef_user" => MU.chef_user,
"mu_user" => MU.mu_user
}
@mommacat = MU::MommaCat.new(
MU.deploy_id,
create: true,
config: @main_config,
environment: @environment,
nocleanup: @nocleanup,
no_artifacts: @no_artifacts,
set_context_to_me: true,
deployment_data: metadata,
mu_user: MU.mu_user
)
MU.setVar("mommacat", @mommacat)
end
@admins.each { |admin|
@mommacat.notify("admins", admin['name'], admin)
}
if @mommacat.numKittens(types: ["Server", "ServerPool"]) > 0
MU::MommaCat.start
end
@deploy_semaphore = Mutex.new
parent_thread_id = Thread.current.object_id
# Run cloud provider-specific deploy meta-artifact creation (ssh keys,
# resource groups, etc)
@mommacat.cloudsUsed.each { |cloud|
cloudclass = MU::Cloud.cloudClass(cloud)
cloudclass.initDeploy(@mommacat)
}
@mommacat.writeDeploySecret
# Kick off threads to create each of our new servers.
@my_threads << Thread.new {
MU.dupGlobals(parent_thread_id)
Thread.current.thread_variable_set("name", "mu_create_container")
# Thread.abort_on_exception = false
MU::Cloud.resource_types.values.each { |data|
if !@main_config[data[:cfg_plural]].nil? and
@main_config[data[:cfg_plural]].size > 0 and
data[:instance].include?(:create)
createResources(@main_config[data[:cfg_plural]], "create")
end
}
}
# Some resources have a "groom" phase too
@my_threads << Thread.new {
MU.dupGlobals(parent_thread_id)
Thread.current.thread_variable_set("name", "mu_groom_container")
# Thread.abort_on_exception = false
MU::Cloud.resource_types.values.each { |data|
if !@main_config[data[:cfg_plural]].nil? and
@main_config[data[:cfg_plural]].size > 0 and
data[:instance].include?(:groom)
createResources(@main_config[data[:cfg_plural]], "groom")
end
}
}
# Poke child threads to make sure they're awake
@my_threads.each do |t|
t.run if t.alive?
end
sleep 5
# Reap child threads.
@my_threads.each do |t|
t.join
end
@mommacat.save!
# XXX Functions have a special behavior where we re-invoke their groom
# methods one more time at the end, so we can guarantee their
# environments are fully populated with all sibling resource idents
# regardless of dependency order. This is, obviously, a disgusting
# hack, and we should revisit our dependency language in the next big
# release.
if !@main_config["functions"].nil? and
@main_config["functions"].size > 0
createResources(@main_config["functions"], "groom")
end
rescue StandardError => e
MU.log e.class.name, MU::ERR, details: caller
@my_threads.each do |t|
if t.object_id != Thread.current.object_id and
t.thread_variable_get("name") != "main_thread" and
t.object_id != parent_thread_id
MU::MommaCat.unlockAll
t.kill
end
end
# If it was a regular old exit, we assume something deeper in already
# handled logging and cleanup for us, and just quietly go away.
if e.class.to_s != "SystemExit"
MU.log e.class.name+": "+e.message, MU::ERR, details: e.backtrace if @verbosity != MU::Logger::SILENT
if !@nocleanup
# Wrap this in a thread to protect the Azure SDK from imploding
# because it mistakenly thinks there's a deadlock.
cleanup_thread = Thread.new {
MU.dupGlobals(parent_thread_id)
Thread.abort_on_exception = false
MU::Cleanup.run(MU.deploy_id, skipsnapshots: true, verbosity: @verbosity, mommacat: @mommacat)
}
cleanup_thread.join
@nocleanup = true # so we don't run this again later
end
end
@reraise_thread.raise MuError, e.inspect, e.backtrace if @reraise_thread
Thread.current.exit
ensure
if @mommacat and @mommacat.numKittens(clouds: ["CloudFormation"]) > 0
MU::Cloud::CloudFormation.writeCloudFormationTemplate(tails: MU::Config.tails, config: @main_config, path: @cloudformation_output, mommacat: @mommacat)
# If we didn't build anything besides CloudFormation, purge useless
# metadata.
if @mommacat.numKittens(clouds: ["CloudFormation"], negate: true) == 0
Thread.list.each do |t|
if t.object_id != Thread.current.object_id and t.thread_variable_get("name") != "main_thread" and t.object_id != parent_thread_id
t.kill
end
end
MU::Cleanup.run(MU.deploy_id, skipcloud: true, verbosity: MU::Logger::SILENT, mommacat: @mommacat)
return
end
end
end
if @mommacat.numKittens(clouds: ["CloudFormation"], negate: true) > 0
if !@mommacat.deployment['servers'].nil? and @mommacat.deployment['servers'].keys.size > 0
# XXX some kind of filter (obey sync_siblings on nodes' configs)
@mommacat.syncLitter(@mommacat.deployment['servers'].keys)
end
deployment = @mommacat.deployment
deployment["deployment_end_time"]=Time.new.strftime("%I:%M %p on %A, %b %d, %Y").to_s;
if MU.myCloud == "AWS"
MU::Cloud::AWS.openFirewallForClients # XXX add the other clouds, or abstract
end
# MU::MommaCat.getLitter(MU.deploy_id, use_cache: false)
if @mommacat.numKittens(types: ["Server", "ServerPool"]) > 0
# MU::MommaCat.syncMonitoringConfig # TODO only invoke if Server or ServerPool actually changed something when @updating
end
end
# Send notifications
sendMail
if @mommacat.numKittens(clouds: ["AWS"]) > 0
MU.log "Generating cost calculation URL for all Amazon Web Services resources."
MU.setLogging(MU::Logger::SILENT)
@environment ||= "dev"
begin
Thread.abort_on_exception = false
t = Thread.new {
Thread.abort_on_exception = true
# I do not understand why this is necessary, but here we are.
Thread.handle_interrupt(MU::Cloud::MuCloudResourceNotImplemented => :never) {
begin
Thread.handle_interrupt(MU::Cloud::MuCloudResourceNotImplemented => :immediate) {
MU.log "Cost calculator not available for this stack, as it uses a resource not implemented in Mu's CloudFormation layer.", MU::DEBUG, verbosity: MU::Logger::NORMAL
Thread.current.exit
}
ensure
end
}
begin
MU.setVar("deploy_id", nil) # make sure we won't ever accidentally blow away the parent deploy
cost_dummy_deploy = MU::Deploy.new(
@environment.dup,
verbosity: MU::Logger::SILENT,
force_cloudformation: true,
cloudformation_path: "/dev/null",
nocleanup: false, # make sure we clean up the cost allocation deploy
stack_conf: @original_config,
reraise_thread: @main_thread,
no_artifacts: true
)
cost_dummy_deploy.run
rescue MU::Cloud::MuCloudFlagNotImplemented, MU::Cloud::MuCloudResourceNotImplemented, MU::MuError => e
# This doesn't seem to get caught and I don't know why and I don't care
# MU.log "Failed to generate AWS cost-calculation URL. Skipping.", MU::WARN, details: "Deployment uses a feature not available in CloudFormation layer.", verbosity: MU::Logger::NORMAL
end
}
t.join
rescue MU::Cloud::MuCloudFlagNotImplemented, MU::Cloud::MuCloudResourceNotImplemented => e
# already handled in the thread what did it
MU.log "Failed to generate AWS cost-calculation URL. Skipping.", MU::WARN, details: "Deployment uses a feature not available in CloudFormation layer.", verbosity: MU::Logger::NORMAL
ensure
MU.setLogging(@verbosity)
MU.log "Deployment #{MU.deploy_id} \"#{MU.handle}\" #{@updating ? "updated" : "complete"}", details: deployment, verbosity: @verbosity
end
else
MU.log "Deployment #{MU.deploy_id} \"#{MU.handle}\" #{@updating ? "updated" : "complete"}", details: deployment, verbosity: @verbosity
end
if MU.summary.size > 0
MU.summary.each { |msg|
puts msg
}
end
@mommacat.sendAdminSlack("Deploy #{MU.deploy_id} \"#{MU.handle}\" #{@updating ? "updated" : "complete"}", msg: MU.summary.join("\n"))
end
private
def sendMail()
$str = ""
if MU.summary.size > 0
MU.summary.each { |msg|
$str += msg+"\n"
}
end
$str += JSON.pretty_generate(@mommacat.deployment)
admin_addrs = @admins.map { |admin|
admin['name'] ||= ""
admin['name']+" <"+admin['email']+">"
}
@admins.each do |data|
message = <<MESSAGE_END
From: #{MU.handle} <#{@fromName}>
To: #{admin_addrs.join(", ")}>
MIME-Version: 1.0
Content-type: text/html
Subject: Mu deployment #{MU.appname} \"#{MU.handle}\" (#{MU.deploy_id}) successfully completed
<br>
<pre>#{$str}</pre>
MESSAGE_END
Net::SMTP.start('localhost') do |smtp|
smtp.send_message message, @fromName, data["email"]
end
end
end
#########################################################################
#########################################################################
def waitOnThreadDependencies(dependent)
if @dependency_threads[dependent].nil?
MU.log "I don't see any dependencies for #{dependent}, moving on", MU::DEBUG
return
else
MU.log "#{dependent} checking/waiting for parent threads...", MU::DEBUG, details: @dependency_threads[dependent]
end
retries = 0
@dependency_threads[dependent].each { |dependent_thread|
found = false
@my_threads.each { |parent_thread|
parent = parent_thread.thread_variable_get("name");
if parent == dependent_thread
found = true
Thread.current.thread_variable_set("waiting_for", parent)
parent_thread.join
Thread.current.thread_variable_set("waiting_for", nil)
MU.log "Thread #{parent} completed, thread #{dependent} proceeding", MU::DEBUG, details: @dependency_threads[dependent]
end
}
# This vile hack brought to you by parent threads spawning after things
# that depend on them. We're working around the slight race condition
# that results. If the parent threads never show up, though, we have
# a more serious problem.
if !found and retries < 5
sleep 5
retries = retries + 1
redo
end
if retries >= 5
raise MuError, "#{dependent} tried five times but never saw #{dependent_thread} in live thread list...\n"+@my_threads.join("\t\n")
end
}
end
#########################################################################
# Helper for setThreadDependencies
#########################################################################
def addDependentThread(parent, child)
@dependency_semaphore.synchronize {
@dependency_threads[child] ||= []
@dependency_threads[child] << parent
@dependency_threads[child].uniq!
MU.log "Thread #{child} will wait on #{parent}", MU::DEBUG, details: @dependency_threads[child]
}
end
#########################################################################
# Tell a service's deploy (and optionally, create) thread to wait on its
# dependent service's create (and optionally, deploy) threads to finish.
# XXX This nomenclature is unreasonably confusing.
#########################################################################
def setThreadDependencies(services)
if services.nil? or services.size < 1
# MU.log "Got nil service list in setThreadDependencies for called from #{caller_locations(1,1)[0].label}", MU::DEBUG
return
end
services.each { |resource|
if !resource["#MU_CLOUDCLASS"]
# pp resource
end
res_type = resource["#MU_CLOUDCLASS"].cfg_name
name = res_type+"_"+resource["name"]
# All resources wait to "groom" until after their own "create" thread
# finishes, and also on the main thread which spawns them (so all
# siblings will exist for dependency checking before we start).
@dependency_threads["#{name}_create"]=["mu_create_container"]
@dependency_threads["#{name}_groom"]=["#{name}_create", "mu_groom_container"]
MU.log "Setting dependencies for #{name}", MU::DEBUG, details: resource["dependencies"]
if !resource["dependencies"].nil? then
resource["dependencies"].each { |dependency|
parent_class = MU::Cloud.loadBaseType(dependency['type'])
parent_type = parent_class.cfg_name
# our groom thread will always need to wait on our parent's create
parent = parent_type+"_"+dependency["name"]+"_create"
addDependentThread(parent, "#{name}_groom")
# if we've explicitly declared each end of the dependency, roll
# with that and don't meddle further
if dependency["my_phase"] and dependency["their_phase"]
parent = parent_type+"_"+dependency["name"]+"_"+dependency["their_phase"]
addDependentThread(parent, name+"_"+dependency["my_phase"])
next
end
# should our creation thread also wait on our parent's create?
if dependency["my_phase"] == "create" and
(resource["#MU_CLOUDCLASS"].waits_on_parent_completion or
parent_class.deps_wait_on_my_creation
)
addDependentThread(parent, "#{name}_create")
end
# how about our groom thread waiting on our parents' grooms?
if (dependency['their_phase'] == "groom" or resource["#MU_CLOUDCLASS"].waits_on_parent_completion) and parent_class.instance_methods(false).include?(:groom)
parent = parent_type+"_"+dependency["name"]+"_groom"
addDependentThread(parent, "#{name}_groom")
if dependency["my_phase"] == "groom" and
(dependency['their_phase'] == "create" or
(!dependency['their_phase'] and
parent_class.deps_wait_on_my_creation or
resource["#MU_CLOUDCLASS"].waits_on_parent_completion)
)
addDependentThread(parent, "#{name}_create")
end
end
}
end
@dependency_threads["#{name}_groom"].concat(["#{name}_create", "mu_groom_container"])
@dependency_threads["#{name}_groom"].uniq!
MU.log "Thread dependencies #{res_type}[#{name}]", MU::DEBUG, details: { "create" => @dependency_threads["#{name}_create"], "groom" => @dependency_threads["#{name}_groom"] } if res_type == "role" and resource['name'] == "dynamostream-to-es"
}
end
#########################################################################
# Kick off a thread to create a resource.
#########################################################################
def createResources(services, mode="create")
return if services.nil?
parent_thread_id = Thread.current.object_id
services.uniq!
services.each do |service|
begin
@my_threads << Thread.new(service) { |myservice|
MU.dupGlobals(parent_thread_id)
threadname = myservice["#MU_CLOUDCLASS"].cfg_name+"_"+myservice["name"]+"_#{mode}"
Thread.current.thread_variable_set("name", threadname)
Thread.current.thread_variable_set("owned_by_mu", true)
# Thread.abort_on_exception = false
waitOnThreadDependencies(threadname)
if myservice["#MU_CLOUDCLASS"].instance_methods(false).include?(:groom) and !myservice['dependencies'].nil? and !myservice['dependencies'].size == 0
if mode == "create"
MU::MommaCat.lock(myservice["#MU_CLOUDCLASS"].cfg_name+"_"+myservice["name"]+"-dependencies")
elsif mode == "groom"
MU::MommaCat.unlock(myservice["#MU_CLOUDCLASS"].cfg_name+"_"+myservice["name"]+"-dependencies")
end
end
MU.log "Launching thread #{threadname}", MU::DEBUG
begin
if myservice['#MUOBJECT'].nil?
if @mommacat
ext_obj = @mommacat.findLitterMate(type: myservice["#MU_CLOUDCLASS"].cfg_plural, name: myservice['name'], credentials: myservice['credentials'], created_only: true, return_all: false, ignore_missing: !@updating)
if @updating and ext_obj
ext_obj.config!(myservice)
end
myservice['#MUOBJECT'] = ext_obj
end
myservice['#MUOBJECT'] ||= myservice["#MU_CLOUDCLASS"].new(mommacat: @mommacat, kitten_cfg: myservice, delayed_save: @updating)
end
rescue RuntimeError => e
# cloud implementations can iterate over these same hashes,
# which can throw this if we catch them at the wrong moment.
# here's your hacky workaround.
if e.message.match(/can't add a new key into hash during iteration/)
MU.log e.message+" in main deploy thread, probably transient", MU::DEBUG
sleep 1
retry
else
raise e
end
rescue StandardError => e
MU::MommaCat.unlockAll
@main_thread.raise MuError, "Error instantiating object from #{myservice["#MU_CLOUDCLASS"]} (#{e.inspect})", e.backtrace
raise e
end
begin
run_this_method = myservice['#MUOBJECT'].method(mode)
rescue StandardError => e
MU::MommaCat.unlockAll
@main_thread.raise MuError, "Error invoking #{myservice["#MUOBJECT"].class.name}.#{mode} for #{myservice['name']} (#{e.inspect})", e.backtrace
return
# raise e
end
begin
MU.log "Checking whether to run #{myservice['#MUOBJECT']}.#{mode} (updating: #{@updating})", MU::DEBUG
if !@updating or mode != "create"
myservice = run_this_method.call
else
# XXX experimental create behavior for --liveupdate flag, only works on a couple of resource types. Inserting new resources into an old deploy is tricky.
opts = {}
if myservice["#MU_CLOUDCLASS"].cfg_name == "loadbalancer"
opts['classic'] = myservice['classic'] ? true : false
end
found = MU::MommaCat.findStray(myservice['cloud'],
myservice["#MU_CLOUDCLASS"].cfg_name,
name: myservice['name'],
credentials: myservice['credentials'],
region: myservice['region'],
deploy_id: @mommacat.deploy_id,
# allow_multi: myservice["#MU_CLOUDCLASS"].has_multiple,
tag_key: "MU-ID",
tag_value: @mommacat.deploy_id,
flags: opts,
dummy_ok: false
)
found = found.delete_if { |x|
x.cloud_id.nil? and x.cloudobj.cloud_id.nil?
}
if found.size == 0
MU.log "#{myservice["#MU_CLOUDCLASS"].name} #{myservice['name']} not found, creating", MU::NOTICE
myservice = run_this_method.call
else
real_descriptor = @mommacat.findLitterMate(type: myservice["#MU_CLOUDCLASS"].cfg_name, name: myservice['name'], created_only: true)
if !real_descriptor
MU.log "Invoking #{run_this_method.to_s} #{myservice['name']} #{myservice['name']}", MU::NOTICE
myservice = run_this_method.call
end
#MU.log "#{myservice["#MU_CLOUDCLASS"].cfg_name} #{myservice['name']}", MU::NOTICE
end
end
rescue ThreadError => e
MU.log "Waiting for threads to complete (#{e.message})", MU::NOTICE
@my_threads.each do |thr|
next if thr.object_id == Thread.current.object_id
thr.join(0.1)
end
@my_threads.reject! { |thr| !thr.alive? }
sleep 10+Random.rand(20)
retry
rescue StandardError => e
MU.log e.inspect, MU::ERR, details: e.backtrace if @verbosity != MU::Logger::SILENT
MU::MommaCat.unlockAll
Thread.list.each do |t|
if t.object_id != Thread.current.object_id and t.thread_variable_get("name") != "main_thread" and t.object_id != parent_thread_id and t.thread_variable_get("owned_by_mu")
t.kill
end
end
if !@nocleanup
MU::Cleanup.run(MU.deploy_id, verbosity: @verbosity, skipsnapshots: true)
@nocleanup = true # so we don't run this again later
end
@main_thread.raise MuError, e.message, e.backtrace
end
MU.purgeGlobals
}
rescue ThreadError => e
MU.log "Waiting for threads to complete (#{e.message})", MU::NOTICE
@my_threads.each do |thr|
next if thr.object_id == Thread.current.object_id
thr.join(0.1)
end
@my_threads.reject! { |thr| !thr.alive? }
sleep 10+Random.rand(20)
retry
end
end
end
end #class
end #module