diff --git a/example-projects/demo3-microservice-in-aws-destroy.py b/example-projects/demo3-microservice-in-aws-destroy.py index b37f498..a7711da 100644 --- a/example-projects/demo3-microservice-in-aws-destroy.py +++ b/example-projects/demo3-microservice-in-aws-destroy.py @@ -10,20 +10,15 @@ from libcloud.compute.types import Provider, NodeState home = expanduser("~") -# reqs: -# services: EC2 (nova, glance, neutron) -# resources: 2 instances (m1.small), 2 elastic ips (1 keypair, 2 security groups) - - # default region # region_name = 'eu-central-1' # region_name = 'ap-south-1' -# AWS Educate only allows us-east-1 see our AWS classroom at https://www.awseducate.com -# e.g., https://www.awseducate.com/student/s/launch-classroom?classroomId=a1v3m000005mNm6AAE - +# AWS Academy Labs only allow us-east-1 see our AWS Academy Lab Guide, https://awsacademy.instructure.com/login/ region_name = 'us-east-1' +# keypairs are kept and not deleted, they do not cost anything anyway + def main(): ########################################################################### diff --git a/example-projects/demo3-microservice-in-aws.py b/example-projects/demo3-microservice-in-aws.py index 1e5eb8e..90541b0 100644 --- a/example-projects/demo3-microservice-in-aws.py +++ b/example-projects/demo3-microservice-in-aws.py @@ -7,9 +7,11 @@ from libcloud.compute.types import Provider home = expanduser("~") -# reqs: -# services: EC2 (nova, glance, neutron) -# resources: 2 instances, 2 elastic ips (1 keypair, 2 security groups) +# requirements: +# services: EC2 +# resources: 2 instances (1 keypair, 2 security groups) +# optionally also elastic ip (comparable to floating ip) offering a persistent public +# IP, but elastic ips are expensive - make sure to delete them after you used them # The image to look for and use for the started instance ubuntu_image_name = 'ubuntu/images/hvm-ssd/ubuntu-bionic-18.04-amd64-server-20210128' @@ -33,7 +35,7 @@ flavor_name = 't2.nano' region_name = 'us-east-1' # starting instances in AWS Academy takes significantly longer compared to paid AWS accounts, allow ~ >2 minutes timeout -timeout = 300 +timeout = 600 def main(): diff --git a/example-projects/demo4-scale-out-lb-in-aws-destroy.py b/example-projects/demo4-scale-out-lb-in-aws-destroy.py index cc95b35..b819f3b 100644 --- a/example-projects/demo4-scale-out-lb-in-aws-destroy.py +++ b/example-projects/demo4-scale-out-lb-in-aws-destroy.py @@ -1,3 +1,6 @@ +import configparser +from os.path import expanduser + # import getpass # import os # import libcloud.security @@ -12,33 +15,17 @@ from libcloud.loadbalancer.base import Member, Algorithm from libcloud.loadbalancer.types import Provider as loadbalancer_Provider from libcloud.loadbalancer.providers import get_driver as loadbalancer_get_driver -# reqs: -# services: EC2 (nova, glance, neutron) -# resources: 2 instances, 2 elastic ips (1 keypair, 2 security groups) - -# The image to look for and use for the started instance -# ubuntu_image_name = 'ubuntu/images/hvm-ssd/ubuntu-bionic-18.04-amd64-server-20200408' -ubuntu_image_id = "ami-085925f297f89fce1" # local ami id for resent ubuntu 18.04 20200408 in us-west-1 - -# The public key to be used for SSH connection, please make sure, that you have the corresponding private key -# -# id_rsa.pub should look like this (standard sshd pubkey format): -# ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAw+J...F3w2mleybgT1w== user@HOSTNAME - -keypair_name = 'srieger-pub' -pub_key_file = '~/.ssh/id_rsa.pub' - -flavor_name = 't2.nano' +home = expanduser("~") # default region # region_name = 'eu-central-1' # region_name = 'ap-south-1' -# AWS Educate only allows us-east-1 see our AWS classroom at https://www.awseducate.com -# e.g., https://www.awseducate.com/student/s/launch-classroom?classroomId=a1v3m000005mNm6AAE - +# AWS Academy Labs only allow us-east-1 see our AWS Academy Lab Guide, https://awsacademy.instructure.com/login/ region_name = 'us-east-1' +# keypairs are kept and not deleted, they do not cost anything anyway + def main(): ########################################################################### @@ -47,17 +34,18 @@ def main(): # ########################################################################### - # see AWS Educate classroom, Account Details + # see AWS Academy Lab for Account Details + # read credentials from file + config = configparser.ConfigParser() + config.read_file(open(home + '/.aws/credentials')) + aws_access_key_id = config['default']['aws_access_key_id'] + aws_secret_access_key = config['default']['aws_secret_access_key'] + aws_session_token = config['default']['aws_session_token'] + + # aws_access_key_id = "ASIAX..." + # aws_secret_access_key = "eGwE12j..." + # aws_session_token = "FwoGZXIvYXdzEK///////////wEaDE..." - # access_id = getpass.win_getpass("Enter your access_id:") - # secret_key = getpass.win_getpass("Enter your secret_key:") - # session_token = getpass.win_getpass("Enter your session_token:") - # access_id = "ASIAU..." - # secret_key = "7lafW..." - # session_token = "IQoJb3JpZ...EMb//..." - access_id = "ASIA5ML7..." - secret_key = "76lAjn..." - session_token = "IQoJb3JpZ2luX2VjEBc..." ########################################################################### # @@ -66,9 +54,9 @@ def main(): ########################################################################### elb_provider = loadbalancer_get_driver(loadbalancer_Provider.ELB) - elb_conn = elb_provider(access_id, - secret_key, - token=session_token, + elb_conn = elb_provider(aws_access_key_id, + aws_secret_access_key, + token=aws_session_token, region=region_name) print("Deleting previously created load balancers in: " + str(elb_conn.list_balancers())) @@ -83,9 +71,9 @@ def main(): ########################################################################### provider = compute_get_driver(compute_Provider.EC2) - conn = provider(key=access_id, - secret=secret_key, - token=session_token, + conn = provider(key=aws_access_key_id, + secret=aws_secret_access_key, + token=aws_session_token, region=region_name) ########################################################################### @@ -102,7 +90,7 @@ def main(): print('Destroying Instance: %s' % instance.name) conn.destroy_node(instance) - # wait until all nodes are destroyed to be able to remove depended security groups + # wait until all nodes are destroyed to be able to remove dependent security groups nodes_still_running = True while nodes_still_running: nodes_still_running = False @@ -115,19 +103,30 @@ def main(): nodes_still_running = True print('There are still instances running, waiting for them to be destroyed...') - # delete security groups, respecting dependencies (hence deleting 'control' and 'services' first) + # delete security groups + for group in conn.ex_list_security_groups(): + # services depends on worker and api, so delete services first... + if group in ['services']: + print('Deleting security group: %s' % group) + conn.ex_delete_security_group(group) + for group in conn.ex_list_security_groups(): - if group in ['control', 'services']: + # control depends on worker, so delete control before worker... + if group in ['control']: print('Deleting security group: %s' % group) conn.ex_delete_security_group(group) - # now we can delete security groups 'api' and 'worker', as 'control' and 'api' depended on them, otherwise AWS will - # throw DependencyViolation: resource has a dependent object for group in conn.ex_list_security_groups(): - if group in ['api', 'worker']: + if group in ['worker', 'api']: print('Deleting security group: %s' % group) conn.ex_delete_security_group(group) + # release elastic ips + for elastic_ip in conn.ex_describe_all_addresses(): + if elastic_ip.instance_id is None: + print('Releasing unused elastic ip %s' % elastic_ip) + conn.ex_release_address(elastic_ip, domain=elastic_ip.domain) + if __name__ == '__main__': main() diff --git a/example-projects/demo4-scale-out-lb-in-aws.py b/example-projects/demo4-scale-out-lb-in-aws.py index 013ffa7..499c7b1 100644 --- a/example-projects/demo4-scale-out-lb-in-aws.py +++ b/example-projects/demo4-scale-out-lb-in-aws.py @@ -1,5 +1,6 @@ -# import getpass -# import os +import configparser +from os.path import expanduser + # import libcloud.security import time @@ -12,13 +13,16 @@ from libcloud.loadbalancer.base import Member, Algorithm from libcloud.loadbalancer.types import Provider as loadbalancer_Provider from libcloud.loadbalancer.providers import get_driver as loadbalancer_get_driver -# reqs: -# services: EC2 (nova, glance, neutron) -# resources: 2 instances, 2 elastic ips (1 keypair, 2 security groups) +home = expanduser("~") + +# requirements: +# services: EC2, ELB +# resources: 2 instances, (1 keypair, 2 security groups), +# 1 (Classic) Elastic Load Balancer, expensive! delete it after you used it! # The image to look for and use for the started instance -# ubuntu_image_name = 'ubuntu/images/hvm-ssd/ubuntu-bionic-18.04-amd64-server-20200408' -ubuntu_image_id = "ami-085925f297f89fce1" # local ami id for resent ubuntu 18.04 20200408 in us-west-1 +ubuntu_image_name = 'ubuntu/images/hvm-ssd/ubuntu-bionic-18.04-amd64-server-20210128' +# TODO: 18.04, currently still needed for faafo, need to port faafo demo app to 20.04 or higher and python3... # The public key to be used for SSH connection, please make sure, that you have the corresponding private key # @@ -26,7 +30,7 @@ ubuntu_image_id = "ami-085925f297f89fce1" # local ami id for resent ubuntu 18.04 # ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAw+J...F3w2mleybgT1w== user@HOSTNAME keypair_name = 'srieger-pub' -pub_key_file = '~/.ssh/id_rsa.pub' +pub_key_file = home + '/.ssh/id_rsa.pub' flavor_name = 't2.nano' @@ -34,11 +38,12 @@ flavor_name = 't2.nano' # region_name = 'eu-central-1' # region_name = 'ap-south-1' -# AWS Educate only allows us-east-1 see our AWS classroom at https://www.awseducate.com -# e.g., https://www.awseducate.com/student/s/launch-classroom?classroomId=a1v3m000005mNm6AAE - +# AWS Academy Labs only allow us-east-1 see our AWS Academy Lab Guide, https://awsacademy.instructure.com/login/ region_name = 'us-east-1' +# starting instances in AWS Academy takes significantly longer compared to paid AWS accounts, allow ~ >2 minutes timeout +timeout = 600 + def main(): ########################################################################### @@ -47,17 +52,19 @@ def main(): # ########################################################################### - # see AWS Educate classroom, Account Details + # see AWS Academy Lab for Account Details + # read credentials from file + config = configparser.ConfigParser() + config.read_file(open(home + '/.aws/credentials')) + aws_access_key_id = config['default']['aws_access_key_id'] + aws_secret_access_key = config['default']['aws_secret_access_key'] + aws_session_token = config['default']['aws_session_token'] + + # hard coded AWS credentials using vars + # aws_access_key_id = "ASIAX..." + # aws_secret_access_key = "WLxxXK+..." + # aws_session_token = "FwoGZXIvYXdzEMb//////////wEaDE5rX.......0SleZ+L75I9iEri9LA4hovWul8HvexhCBK8.......................Ae/T+VkUbcQRtJEDwg+gYCABuk0JlSj5Wk7YA65r3BSNJXZFpkhbek6VBjvE/cEt5fKZEhENcdFxjAcAJLd6bOWi/oGXU5e3PX3mcXgm0oJpz6h3wqD1LvSDtw5GDwn0BHiF1Mu.......................cm/VukK5F" - # access_id = getpass.win_getpass("Enter your access_id:") - # secret_key = getpass.win_getpass("Enter your secret_key:") - # session_token = getpass.win_getpass("Enter your session_token:") - # access_id = "ASIAU..." - # secret_key = "7lafW..." - # session_token = "IQoJb3JpZ...EMb//..." - access_id = "ASIA..." - secret_key = "76lAj..." - session_token = "IQoJ..." ########################################################################### # @@ -66,9 +73,9 @@ def main(): ########################################################################### provider = compute_get_driver(compute_Provider.EC2) - conn = provider(key=access_id, - secret=secret_key, - token=session_token, + conn = provider(key=aws_access_key_id, + secret=aws_secret_access_key, + token=aws_session_token, region=region_name) ########################################################################### @@ -77,15 +84,32 @@ def main(): # ########################################################################### + print("Search for AMI...") + image = conn.list_images(ex_filters={"name": ubuntu_image_name})[0] + print("Using image: %s" % image) + # print("Fetching images (AMI) list from AWS region. This will take a lot of seconds (AWS has a very long list of " # "supported operating systems and versions)... please be patient...") - # images = conn.list_images() + # image = '' + # for img in images: + # # if img.name == ubuntu_image_name: + # if img.extra['owner_alias'] == 'amazon': + # print(img) + # if img.id == ubuntu_image_name: + # image = img + # fetch/select the image referenced with ubuntu_image_name above # image = [i for i in images if i.name == ubuntu_image_name][0] # print(image) - # selecting the image based on defined AMI id - image = NodeImage(id=ubuntu_image_id, name=None, driver=conn) + # select image directly to save time, as retrieving the image list takes several minutes now, + # need to change ami id here if updated or for other regions, id is working for course in + # summer term 2022, in region: us-east-1 and pointing to ubuntu 18.04 used in the instance wizard, + # to update AMI id use the create instance wizard and copy amd64 image id for ubuntu 18.04 in the + # desired region + # image = NodeImage(id="ami-0e472ba40eb589f49", + # name=ubuntu_image_name, + # driver="hvm") flavors = conn.list_sizes() flavor = [s for s in flavors if s.id == flavor_name][0] @@ -132,7 +156,7 @@ def main(): print('Destroying Instance: %s' % instance.name) conn.destroy_node(instance) - # wait until all nodes are destroyed to be able to remove depended security groups + # wait until all nodes are destroyed to be able to remove dependent security groups nodes_still_running = True while nodes_still_running: nodes_still_running = False @@ -234,7 +258,7 @@ def main(): # Thanks to Stefan Friedmann for finding this fix ;) userdata_service = '''#!/usr/bin/env bash - curl -L -s https://gogs.informatik.hs-fulda.de/srieger/cloud-computing-msc-ai-examples/raw/master/faafo/contrib/install.sh | bash -s -- \ + curl -L -s https://gogs.informatik.hs-fulda.de/srieger/cloud-computing-msc-ai-examples/raw/master/faafo/contrib/install-aws.sh | bash -s -- \ -i database -i messaging rabbitmqctl add_user faafo guest rabbitmqctl set_user_tags faafo administrator @@ -258,7 +282,7 @@ def main(): ########################################################################### userdata_api = '''#!/usr/bin/env bash - curl -L -s https://gogs.informatik.hs-fulda.de/srieger/cloud-computing-msc-ai-examples/raw/master/faafo/contrib/install.sh | bash -s -- \ + curl -L -s https://gogs.informatik.hs-fulda.de/srieger/cloud-computing-msc-ai-examples/raw/master/faafo/contrib/install-aws.sh | bash -s -- \ -i faafo -r api -m 'amqp://faafo:guest@%(services_ip)s:5672/' \ -d 'mysql+pymysql://faafo:password@%(services_ip)s:3306/faafo' ''' % {'services_ip': services_ip} @@ -291,7 +315,7 @@ def main(): ########################################################################### userdata_worker = '''#!/usr/bin/env bash - curl -L -s https://gogs.informatik.hs-fulda.de/srieger/cloud-computing-msc-ai-examples/raw/master/faafo/contrib/install.sh | bash -s -- \ + curl -L -s https://gogs.informatik.hs-fulda.de/srieger/cloud-computing-msc-ai-examples/raw/master/faafo/contrib/install-aws.sh | bash -s -- \ -i faafo -r worker -e 'http://%(api_1_ip)s' -m 'amqp://faafo:guest@%(services_ip)s:5672/' ''' % {'api_1_ip': api_1_ip, 'services_ip': services_ip} @@ -335,9 +359,9 @@ def main(): ########################################################################### elb_provider = loadbalancer_get_driver(loadbalancer_Provider.ELB) - elb_conn = elb_provider(access_id, - secret_key, - token=session_token, + elb_conn = elb_provider(aws_access_key_id, + aws_secret_access_key, + token=aws_session_token, region=region_name) print("Deleting previously created load balancers in: " + str(elb_conn.list_balancers()))