@ -22,6 +22,7 @@ home = expanduser("~")
# The image to look for and use for the started instance # The image to look for and use for the started instance
ubuntu_image_name = ' ubuntu/images/hvm-ssd/ubuntu-bionic-18.04-amd64-server-20210128 ' ubuntu_image_name = ' ubuntu/images/hvm-ssd/ubuntu-bionic-18.04-amd64-server-20210128 '
# ubuntu_image_name = 'ubuntu/images/hvm-ssd/ubuntu-bionic-18.04-amd64-server-20210128'
# TODO: 18.04, currently still needed for faafo, need to port faafo demo app to 20.04 or higher and python3... # TODO: 18.04, currently still needed for faafo, need to port faafo demo app to 20.04 or higher and python3...
# The public key to be used for SSH connection, please make sure, that you have the corresponding private key # The public key to be used for SSH connection, please make sure, that you have the corresponding private key
@ -32,14 +33,16 @@ ubuntu_image_name = 'ubuntu/images/hvm-ssd/ubuntu-bionic-18.04-amd64-server-2021
keypair_name = ' srieger-pub ' keypair_name = ' srieger-pub '
pub_key_file = home + ' /.ssh/id_rsa.pub ' pub_key_file = home + ' /.ssh/id_rsa.pub '
flavor_name = ' t2.nano '
# flavor_name = 't2.nano'
flavor_name = ' t2.micro '
# default region # default region
# region_name = 'eu-central-1' # region_name = 'eu-central-1'
# region_name = 'ap-south-1' # region_name = 'ap-south-1'
# AWS Academy Labs only allow us-east-1 see our AWS Academy Lab Guide, https://awsacademy.instructure.com/login/
# AWS Academy Labs only allow us-east-1 and us-west-1 see our AWS Academy Lab Guide, https://awsacademy.instructure.com/login/
region_name = ' us-east-1 ' region_name = ' us-east-1 '
# region_name = 'us-west-1'
# starting instances in AWS Academy takes significantly longer compared to paid AWS accounts, allow ~ >2 minutes timeout # starting instances in AWS Academy takes significantly longer compared to paid AWS accounts, allow ~ >2 minutes timeout
timeout = 600 timeout = 600
@ -84,9 +87,9 @@ def main():
# #
########################################################################### ###########################################################################
print ( " Search for AMI... " )
image = conn . list_images ( ex_filters = { " name " : ubuntu_image_name } ) [ 0 ]
print ( " Using image: %s " % image )
# print("Search for AMI..." )
# image = conn.list_images(ex_filters={"name": ubuntu_image_name})[0 ]
# print("Using image: %s" % image )
# print("Fetching images (AMI) list from AWS region. This will take a lot of seconds (AWS has a very long list of " # print("Fetching images (AMI) list from AWS region. This will take a lot of seconds (AWS has a very long list of "
# "supported operating systems and versions)... please be patient...") # "supported operating systems and versions)... please be patient...")
@ -98,18 +101,27 @@ def main():
# if img.id == ubuntu_image_name: # if img.id == ubuntu_image_name:
# image = img # image = img
# fetch/select the image referenced with ubuntu_image_name above
# image = [i for i in images if i.name == ubuntu_image_name][0]
# print(image)
# select image directly to save time, as retrieving the image list takes several minutes now, # select image directly to save time, as retrieving the image list takes several minutes now,
# need to change ami id here if updated or for other regions, id is working for course in # need to change ami id here if updated or for other regions, id is working for course in
# summer term 2022, in region: us-east-1 and pointing to ubuntu 18.04 used in the instance wizard, # summer term 2022, in region: us-east-1 and pointing to ubuntu 18.04 used in the instance wizard,
# to update AMI id use the create instance wizard and copy amd64 image id for ubuntu 18.04 in the # to update AMI id use the create instance wizard and copy amd64 image id for ubuntu 18.04 in the
# desired region # desired region
# image = NodeImage(id="ami-0e472ba40eb589f49",
# name=ubuntu_image_name,
# driver="hvm")
#
print ( " Selecting AMI... " )
# us-east-1 examples as of 9.5.2022:
#
# Canonical, Ubuntu, 18.04 LTS, amd64 bionic image build on 2022-04-11
image_id = " ami-005de95e8ff495156 "
#
# Canonical, Ubuntu, 20.04 LTS, amd64 focal image build on 2022-04-19
# image_id = "ami-0c4f7023847b90238"
#
# Canonical, Ubuntu, 22.04 LTS, amd64 jammy image build on 2022-04-20
# image_id = "ami-09d56f8956ab235b3"
#
image = conn . list_images ( ex_image_ids = [ image_id ] ) [ 0 ]
print ( " Using image: %s " % image )
flavors = conn . list_sizes ( ) flavors = conn . list_sizes ( )
flavor = [ s for s in flavors if s . id == flavor_name ] [ 0 ] flavor = [ s for s in flavors if s . id == flavor_name ] [ 0 ]
@ -164,10 +176,14 @@ def main():
instances = conn . list_nodes ( ) instances = conn . list_nodes ( )
for instance in instances : for instance in instances :
# if we see any demo instances still running continue to wait for them to stop # if we see any demo instances still running continue to wait for them to stop
if instance . name in [ ' all-in-one ' , ' app-worker-1 ' , ' app-worker-2 ' , ' app-controller ' , ' app-services ' ] :
if instance . name in [ ' all-in-one ' , ' app-worker-1 ' , ' app-worker-2 ' , ' app-worker-3 ' , ' app-controller ' ,
' app-services ' , ' app-api-1 ' , ' app-api-2 ' ] :
if instance . state is not NodeState . TERMINATED : if instance . state is not NodeState . TERMINATED :
nodes_still_running = True nodes_still_running = True
print ( ' There are still instances running, waiting for them to be destroyed... ' )
if nodes_still_running is True :
print ( ' There are still instances running, waiting for them to be destroyed... ' )
else :
print ( ' No instances running ' )
# delete security groups, respecting dependencies (hence deleting 'control' and 'services' first) # delete security groups, respecting dependencies (hence deleting 'control' and 'services' first)
for group in conn . ex_list_security_groups ( ) : for group in conn . ex_list_security_groups ( ) :
@ -233,7 +249,9 @@ def main():
services_security_group_id = services_security_group_result [ ' group_id ' ] services_security_group_id = services_security_group_result [ ' group_id ' ]
conn . ex_authorize_security_group_ingress ( services_security_group_id , 22 , 22 , cidr_ips = [ ' 0.0.0.0/0 ' ] , conn . ex_authorize_security_group_ingress ( services_security_group_id , 22 , 22 , cidr_ips = [ ' 0.0.0.0/0 ' ] ,
protocol = ' tcp ' ) protocol = ' tcp ' )
conn . ex_authorize_security_group_ingress ( services_security_group_id , 3306 , 3306 , cidr_ips = [ ' 0.0.0.0/0 ' ] ,
# conn.ex_authorize_security_group_ingress(services_security_group_id, 3306, 3306, cidr_ips=['0.0.0.0/0'],
# group_pairs=[{'group_id': api_security_group_id}], protocol='tcp')
conn . ex_authorize_security_group_ingress ( services_security_group_id , 3306 , 3306 ,
group_pairs = [ { ' group_id ' : api_security_group_id } ] , protocol = ' tcp ' ) group_pairs = [ { ' group_id ' : api_security_group_id } ] , protocol = ' tcp ' )
conn . ex_authorize_security_group_ingress ( services_security_group_id , 5672 , 5672 , conn . ex_authorize_security_group_ingress ( services_security_group_id , 5672 , 5672 ,
group_pairs = [ { ' group_id ' : worker_security_group_id } ] , protocol = ' tcp ' ) group_pairs = [ { ' group_id ' : worker_security_group_id } ] , protocol = ' tcp ' )
@ -245,6 +263,12 @@ def main():
for security_group in conn . ex_list_security_groups ( ) : for security_group in conn . ex_list_security_groups ( ) :
print ( security_group ) print ( security_group )
# get availability zones
az = conn . list_locations ( )
print ( az )
########################################################################### ###########################################################################
# #
# create app-services instance (database & messaging) (Amazon AWS EC2) # create app-services instance (database & messaging) (Amazon AWS EC2)
@ -258,7 +282,7 @@ def main():
# Thanks to Stefan Friedmann for finding this fix ;) # Thanks to Stefan Friedmann for finding this fix ;)
userdata_service = ''' #!/usr/bin/env bash userdata_service = ''' #!/usr/bin/env bash
curl - L - s https : / / gogs . informatik . hs - fulda . de / srieger / cloud - computing - msc - ai - examples / raw / master / faafo / contrib / install - aws . sh | bash - s - - \
curl - L - s https : / / gogs . informatik . hs - fulda . de / srieger / cloud - computing - msc - ai - examples / raw / master / faafo / contrib / install . sh | bash - s - - \
- i database - i messaging - i database - i messaging
rabbitmqctl add_user faafo guest rabbitmqctl add_user faafo guest
rabbitmqctl set_user_tags faafo administrator rabbitmqctl set_user_tags faafo administrator
@ -266,7 +290,8 @@ def main():
''' '''
print ( ' Starting new app-services instance and wait until it is running... ' ) print ( ' Starting new app-services instance and wait until it is running... ' )
instance_services = conn . create_node ( name = ' app-services ' ,
instance_services = conn . create_node ( location = az [ 0 ] ,
name = ' app-services ' ,
image = image , image = image ,
size = flavor , size = flavor ,
ex_keyname = keypair_name , ex_keyname = keypair_name ,
@ -282,13 +307,14 @@ def main():
########################################################################### ###########################################################################
userdata_api = ''' #!/usr/bin/env bash userdata_api = ''' #!/usr/bin/env bash
curl - L - s https : / / gogs . informatik . hs - fulda . de / srieger / cloud - computing - msc - ai - examples / raw / master / faafo / contrib / install - aws . sh | bash - s - - \
curl - L - s https : / / gogs . informatik . hs - fulda . de / srieger / cloud - computing - msc - ai - examples / raw / master / faafo / contrib / install . sh | bash - s - - \
- i faafo - r api - m ' amqp://faafo:guest@ %(services_ip)s :5672/ ' \ - i faafo - r api - m ' amqp://faafo:guest@ %(services_ip)s :5672/ ' \
- d ' mysql+pymysql://faafo:password@ %(services_ip)s :3306/faafo ' - d ' mysql+pymysql://faafo:password@ %(services_ip)s :3306/faafo '
''' % { ' services_ip ' : services_ip} ''' % { ' services_ip ' : services_ip}
print ( ' Starting new app-api-1 instance and wait until it is running... ' ) print ( ' Starting new app-api-1 instance and wait until it is running... ' )
instance_api_1 = conn . create_node ( name = ' app-api-1 ' ,
instance_api_1 = conn . create_node ( location = az [ 0 ] ,
name = ' app-api-1 ' ,
image = image , image = image ,
size = flavor , size = flavor ,
ex_keyname = keypair_name , ex_keyname = keypair_name ,
@ -296,7 +322,8 @@ def main():
ex_security_groups = [ " api " ] ) ex_security_groups = [ " api " ] )
print ( ' Starting new app-api-2 instance and wait until it is running... ' ) print ( ' Starting new app-api-2 instance and wait until it is running... ' )
instance_api_2 = conn . create_node ( name = ' app-api-2 ' ,
instance_api_2 = conn . create_node ( location = az [ 1 ] ,
name = ' app-api-2 ' ,
image = image , image = image ,
size = flavor , size = flavor ,
ex_keyname = keypair_name , ex_keyname = keypair_name ,
@ -305,8 +332,11 @@ def main():
instance_api_1 = conn . wait_until_running ( nodes = [ instance_api_1 ] , timeout = 120 , ssh_interface = ' public_ips ' ) instance_api_1 = conn . wait_until_running ( nodes = [ instance_api_1 ] , timeout = 120 , ssh_interface = ' public_ips ' )
api_1_ip = instance_api_1 [ 0 ] [ 0 ] . private_ips [ 0 ] api_1_ip = instance_api_1 [ 0 ] [ 0 ] . private_ips [ 0 ]
print ( " app-api-1 public ip: " + instance_api_1 [ 0 ] [ 1 ] [ 0 ] )
instance_api_2 = conn . wait_until_running ( nodes = [ instance_api_2 ] , timeout = 120 , ssh_interface = ' public_ips ' ) instance_api_2 = conn . wait_until_running ( nodes = [ instance_api_2 ] , timeout = 120 , ssh_interface = ' public_ips ' )
# currently only api_1_ip is used
api_2_ip = instance_api_2 [ 0 ] [ 0 ] . private_ips [ 0 ] api_2_ip = instance_api_2 [ 0 ] [ 0 ] . private_ips [ 0 ]
print ( " app-api-2 public ip: " + instance_api_2 [ 0 ] [ 1 ] [ 0 ] )
########################################################################### ###########################################################################
# #
@ -315,7 +345,7 @@ def main():
########################################################################### ###########################################################################
userdata_worker = ''' #!/usr/bin/env bash userdata_worker = ''' #!/usr/bin/env bash
curl - L - s https : / / gogs . informatik . hs - fulda . de / srieger / cloud - computing - msc - ai - examples / raw / master / faafo / contrib / install - aws . sh | bash - s - - \
curl - L - s https : / / gogs . informatik . hs - fulda . de / srieger / cloud - computing - msc - ai - examples / raw / master / faafo / contrib / install . sh | bash - s - - \
- i faafo - r worker - e ' http:// %(api_1_ip)s ' - m ' amqp://faafo:guest@ %(services_ip)s :5672/ ' - i faafo - r worker - e ' http:// %(api_1_ip)s ' - m ' amqp://faafo:guest@ %(services_ip)s :5672/ '
''' % { ' api_1_ip ' : api_1_ip, ' services_ip ' : services_ip} ''' % { ' api_1_ip ' : api_1_ip, ' services_ip ' : services_ip}
@ -325,14 +355,16 @@ def main():
# ''' % {'api_2_ip': api_2_ip, 'services_ip': services_ip} # ''' % {'api_2_ip': api_2_ip, 'services_ip': services_ip}
print ( ' Starting new app-worker-1 instance and wait until it is running... ' ) print ( ' Starting new app-worker-1 instance and wait until it is running... ' )
instance_worker_1 = conn . create_node ( name = ' app-worker-1 ' ,
instance_worker_1 = conn . create_node ( location = az [ 0 ] ,
name = ' app-worker-1 ' ,
image = image , size = flavor , image = image , size = flavor ,
ex_keyname = keypair_name , ex_keyname = keypair_name ,
ex_userdata = userdata_worker , ex_userdata = userdata_worker ,
ex_security_groups = [ " worker " ] ) ex_security_groups = [ " worker " ] )
print ( ' Starting new app-worker-2 instance and wait until it is running... ' ) print ( ' Starting new app-worker-2 instance and wait until it is running... ' )
instance_worker_2 = conn . create_node ( name = ' app-worker-2 ' ,
instance_worker_2 = conn . create_node ( location = az [ 1 ] ,
name = ' app-worker-2 ' ,
image = image , size = flavor , image = image , size = flavor ,
ex_keyname = keypair_name , ex_keyname = keypair_name ,
ex_userdata = userdata_worker , ex_userdata = userdata_worker ,
@ -367,6 +399,7 @@ def main():
print ( " Deleting previously created load balancers in: " + str ( elb_conn . list_balancers ( ) ) ) print ( " Deleting previously created load balancers in: " + str ( elb_conn . list_balancers ( ) ) )
for loadbalancer in elb_conn . list_balancers ( ) : for loadbalancer in elb_conn . list_balancers ( ) :
if loadbalancer . name == " lb1 " : if loadbalancer . name == " lb1 " :
print ( " Deleting Load Balancer: " + str ( loadbalancer ) )
elb_conn . destroy_balancer ( loadbalancer ) elb_conn . destroy_balancer ( loadbalancer )
# get suffix (a, b, c, ...) from all availability zones, available in the selected region # get suffix (a, b, c, ...) from all availability zones, available in the selected region
