added image search, expand user home dir based on platform, removed elastic ips to save budget, load AWS credentials file, fixed ÓOM in AWS m1.tiny when installing faafo reqs using pip, increased timeout

example-projects/demo3-microservice-in-aws-destroy.py

@@ -1,4 +1,6 @@
-import getpass
+import configparser
+from os.path import expanduser
+
 # import os
 # import libcloud.security
 
@@ -6,6 +8,8 @@ import time
 from libcloud.compute.providers import get_driver
 from libcloud.compute.types import Provider, NodeState
 
+home = expanduser("~")
+
 # reqs:
 #   services: EC2 (nova, glance, neutron)
 #   resources: 2 instances (m1.small), 2 elastic ips (1 keypair, 2 security groups)
@@ -28,14 +32,18 @@ def main():
     #
     ###########################################################################
 
-    # see AWS Educate classroom, Account Details
+    # see AWS Academy Lab for Account Details
+    # read credentials from file
+    config = configparser.ConfigParser()
+    config.read_file(open(home + '/.aws/credentials'))
+    aws_access_key_id = config['default']['aws_access_key_id']
+    aws_secret_access_key = config['default']['aws_secret_access_key']
+    aws_session_token = config['default']['aws_session_token']
+
+    # aws_access_key_id = "ASIAX..."
+    # aws_secret_access_key = "eGwE12j..."
+    # aws_session_token = "FwoGZXIvYXdzEK///////////wEaDE..."
 
-    # access_id = getpass.win_getpass("Enter your access_id:")
-    # secret_key = getpass.win_getpass("Enter your secret_key:")
-    # session_token = getpass.win_getpass("Enter your session_token:")
-    access_id = "ASIAX..."
-    secret_key = "eGwE12j..."
-    session_token = "FwoGZXIvYXdzEK///////////wEaDE..."
 
     ###########################################################################
     #
@@ -44,9 +52,9 @@ def main():
     ###########################################################################
 
     provider = get_driver(Provider.EC2)
-    conn = provider(access_id,
-                    secret_key,
-                    token=session_token,
+    conn = provider(aws_access_key_id,
+                    aws_secret_access_key,
+                    token=aws_session_token,
                     region=region_name)
 
     ###########################################################################
@@ -59,6 +67,7 @@ def main():
     for instance in conn.list_nodes():
         if instance.name in ['all-in-one', 'app-worker-1', 'app-worker-2', 'app-worker-3', 'app-controller',
                              'app-services', 'app-api-1', 'app-api-2']:
+            if instance.state.value != 'terminated':
                 print('Destroying Instance: %s' % instance.name)
                 conn.destroy_node(instance)
 
@@ -78,7 +87,19 @@ def main():
 
     # delete security groups
     for group in conn.ex_list_security_groups():
-        if group in ['control', 'worker', 'api', 'services']:
+        # services depends on worker and api, so delete services first...
+        if group in ['services']:
+            print('Deleting security group: %s' % group)
+            conn.ex_delete_security_group(group)
+
+    for group in conn.ex_list_security_groups():
+        # control depends on worker, so delete control before worker...
+        if group in ['control']:
+            print('Deleting security group: %s' % group)
+            conn.ex_delete_security_group(group)
+
+    for group in conn.ex_list_security_groups():
+        if group in ['worker', 'api']:
             print('Deleting security group: %s' % group)
             conn.ex_delete_security_group(group)
 

example-projects/demo3-microservice-in-aws.py

@@ -1,16 +1,19 @@
-import getpass
-# import os
+import configparser
+from os.path import expanduser
+
 from libcloud.compute.base import NodeImage
 from libcloud.compute.providers import get_driver
 from libcloud.compute.types import Provider
 
+home = expanduser("~")
+
 # reqs:
 #   services: EC2 (nova, glance, neutron)
 #   resources: 2 instances, 2 elastic ips (1 keypair, 2 security groups)
 
 # The image to look for and use for the started instance
-ubuntu_image_name = 'ubuntu/images/hvm-ssd/ubuntu-bionic-18.04-amd64-server-20200408'
-# ubuntu_image_id = "ami-0e342d72b12109f91" # local ami id for resent ubuntu 18.04 20200408 in eu-central-1
+ubuntu_image_name = 'ubuntu/images/hvm-ssd/ubuntu-bionic-18.04-amd64-server-20210128'
+# TODO: 18.04, currently still needed for faafo, need to port faafo demo app to 20.04 or higher and python3...
 
 # The public key to be used for SSH connection, please make sure, that you have the corresponding private key
 #
@@ -18,7 +21,7 @@ ubuntu_image_name = 'ubuntu/images/hvm-ssd/ubuntu-bionic-18.04-amd64-server-2020
 # ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAw+J...F3w2mleybgT1w== user@HOSTNAME
 
 keypair_name = 'srieger-pub'
-pub_key_file = '~/.ssh/id_rsa.pub'
+pub_key_file = home + '/.ssh/id_rsa.pub'
 
 flavor_name = 't2.nano'
 
@@ -26,11 +29,12 @@ flavor_name = 't2.nano'
 # region_name = 'eu-central-1'
 # region_name = 'ap-south-1'
 
-# AWS Educate only allows us-east-1 see our AWS classroom at https://www.awseducate.com
-# e.g., https://www.awseducate.com/student/s/launch-classroom?classroomId=a1v3m000005mNm6AAE
-
+# AWS Academy Labs only allow us-east-1 see our AWS Academy Lab Guide, https://awsacademy.instructure.com/login/
 region_name = 'us-east-1'
 
+# starting instances in AWS Academy takes significantly longer compared to paid AWS accounts, allow ~ >2 minutes timeout
+timeout = 300
+
 
 def main():
     ###########################################################################
@@ -39,14 +43,18 @@ def main():
     #
     ###########################################################################
 
-    # see AWS Educate classroom, Account Details
+    # see AWS Academy Lab for Account Details
+    # read credentials from file
+    config = configparser.ConfigParser()
+    config.read_file(open(home + '/.aws/credentials'))
+    aws_access_key_id = config['default']['aws_access_key_id']
+    aws_secret_access_key = config['default']['aws_secret_access_key']
+    aws_session_token = config['default']['aws_session_token']
 
-    # access_id = getpass.win_getpass("Enter your access_id:")
-    # secret_key = getpass.win_getpass("Enter your secret_key:")
-    # session_token = getpass.win_getpass("Enter your session_token:")
-    access_id = "ASIAX..."
-    secret_key = "WLxxXK+..."
-    session_token = "FwoGZXIvYXdzEMb//////////wEaDE5rX.......0SleZ+L75I9iEri9LA4hovWul8HvexhCBK8.......................Ae/T+VkUbcQRtJEDwg+gYCABuk0JlSj5Wk7YA65r3BSNJXZFpkhbek6VBjvE/cEt5fKZEhENcdFxjAcAJLd6bOWi/oGXU5e3PX3mcXgm0oJpz6h3wqD1LvSDtw5GDwn0BHiF1Mu.......................cm/VukK5F"
+    # hard coded AWS credentials using vars
+    # aws_access_key_id = "ASIAX..."
+    # aws_secret_access_key = "WLxxXK+..."
+    # aws_session_token = "FwoGZXIvYXdzEMb//////////wEaDE5rX.......0SleZ+L75I9iEri9LA4hovWul8HvexhCBK8.......................Ae/T+VkUbcQRtJEDwg+gYCABuk0JlSj5Wk7YA65r3BSNJXZFpkhbek6VBjvE/cEt5fKZEhENcdFxjAcAJLd6bOWi/oGXU5e3PX3mcXgm0oJpz6h3wqD1LvSDtw5GDwn0BHiF1Mu.......................cm/VukK5F"
 
     ###########################################################################
     #
@@ -55,9 +63,9 @@ def main():
     ###########################################################################
 
     provider = get_driver(Provider.EC2)
-    conn = provider(access_id,
-                    secret_key,
-                    token=session_token,
+    conn = provider(aws_access_key_id,
+                    aws_secret_access_key,
+                    token=aws_session_token,
                     region=region_name)
 
     ###########################################################################
@@ -66,10 +74,12 @@ def main():
     #
     ###########################################################################
 
+    print("Search for AMI...")
+    image = conn.list_images(ex_filters={"name": ubuntu_image_name})[0]
+    print("Using image: %s" % image)
+
     # print("Fetching images (AMI) list from AWS region. This will take a lot of seconds (AWS has a very long list of "
     #       "supported operating systems and versions)... please be patient...")
-    # images = conn.list_images()
-
     # image = ''
     # for img in images:
     #   # if img.name == ubuntu_image_name:
@@ -79,7 +89,6 @@ def main():
     #       image = img
 
     # fetch/select the image referenced with ubuntu_image_name above
-
     # image = [i for i in images if i.name == ubuntu_image_name][0]
     # print(image)
 
@@ -88,9 +97,9 @@ def main():
     # summer term 2022, in region: us-east-1 and pointing to ubuntu 18.04 used in the instance wizard,
     # to update AMI id use the create instance wizard and copy amd64 image id for ubuntu 18.04 in the
     # desired region
-    image = NodeImage(id="ami-0e472ba40eb589f49",
-                      name="ubuntu/images/hvm-ssd/ubuntu-bionic-18.04-amd64-server-20211129",
-                      driver="hvm")
+    # image = NodeImage(id="ami-0e472ba40eb589f49",
+    #                   name=ubuntu_image_name,
+    #                   driver="hvm")
 
     flavors = conn.list_sizes()
     flavor = [s for s in flavors if s.id == flavor_name][0]
@@ -183,14 +192,16 @@ def main():
     # Thanks to Stefan Friedmann for finding this fix ;)
 
     userdata = '''#!/usr/bin/env bash
-    curl -L -s https://gogs.informatik.hs-fulda.de/srieger/cloud-computing-msc-ai-examples/raw/master/faafo/contrib/install.sh | bash -s -- \
+    curl -L -s https://gogs.informatik.hs-fulda.de/srieger/cloud-computing-msc-ai-examples/raw/master/faafo/contrib/install-aws.sh | bash -s -- \
         -i messaging -i faafo -r api
     rabbitmqctl add_user faafo guest
     rabbitmqctl set_user_tags faafo administrator
     rabbitmqctl set_permissions -p / faafo ".*" ".*" ".*"
     '''
 
-    print('Starting new app-controller instance and wait until it is running...')
+    print('Starting new app-controller instance and wait until it is running (can take several minutes in free AWS'
+          ' academy accounts compared to seconds when using a regular paid AWS account), timeout %i seconds...'
+          % timeout)
     instance_controller_1 = conn.create_node(name='app-controller',
                                              image=image,
                                              size=flavor,
@@ -198,7 +209,8 @@ def main():
                                              ex_userdata=userdata,
                                              ex_security_groups=[controller_security_group_name])
 
-    conn.wait_until_running(nodes=[instance_controller_1], timeout=120, ssh_interface='public_ips')
+    wait_until_running_result = conn.wait_until_running(nodes=[instance_controller_1], timeout=timeout, ssh_interface='public_ips')
+    instance_controller_1, node_addresses = wait_until_running_result[0]
 
     ###########################################################################
     #
@@ -206,20 +218,22 @@ def main():
     #
     ###########################################################################
 
-    # AWS offers elastic ips, that have the same function as floating IPs in OpenStack. However elastic IPs cost money,
-    # and instances typically already have public IP in AWS, what a luxury ;)
-    print('Checking for unused Elastic IP...')
-    unused_elastic_ip = None
-    for elastic_ip in conn.ex_describe_all_addresses():
-        if not elastic_ip.instance_id:
-            unused_elastic_ip = elastic_ip
-            break
-
-    if not unused_elastic_ip:
-        print('Allocating new Elastic IP')
-        unused_elastic_ip = conn.ex_allocate_address()
-    conn.ex_associate_address_with_node(instance_controller_1, unused_elastic_ip)
-    print('Controller Application will be deployed to http://%s' % unused_elastic_ip.ip)
+    # AWS offers elastic ips, that have the same function as floating IPs in OpenStack. However, elastic IPs cost money,
+    # and instances typically already have public IP in AWS, what a luxury ;) so I commented out elastic IP creation to
+    # save your AWS academy budget
+
+    # print('Checking for unused Elastic IP...')
+    # unused_elastic_ip = None
+    # for elastic_ip in conn.ex_describe_all_addresses():
+    #     if not elastic_ip.instance_id:
+    #         unused_elastic_ip = elastic_ip
+    #         break
+    #
+    # if not unused_elastic_ip:
+    #     print('Allocating new Elastic IP')
+    #     unused_elastic_ip = conn.ex_allocate_address()
+    # conn.ex_associate_address_with_node(instance_controller_1, unused_elastic_ip)
+    # print('Controller Application will be deployed to http://%s' % unused_elastic_ip.ip)
 
     ###########################################################################
     #
@@ -227,12 +241,14 @@ def main():
     #
     ###########################################################################
 
-    # instance should not have a public ip? floating ips are assigned later
-    # instance_controller_1 = conn.ex_get_node_details(instance_controller_1.id)
-    # if instance_controller_1.public_ips:
-    #    ip_controller = instance_controller_1.public_ips[0]
-    # else:
-    ip_controller = instance_controller_1.private_ips[0]
+    # instance_controller_1 = conn.list_nodes(ex_node_ids=instance_controller_1.id)
+    public_ip_controller = instance_controller_1.public_ips[0]
+    print('Controller Application %s will be reachable after cloud-init has run at http://%s, you can also connect'
+           % (instance_controller_1.id, public_ip_controller) +
+          ' via ssh ubuntu@%s' % public_ip_controller)
+
+    # get private IP of instance to use in worker/service instances to connect to the controller
+    private_ip_controller = instance_controller_1.private_ips[0]
 
     ###########################################################################
     #
@@ -241,9 +257,9 @@ def main():
     ###########################################################################
 
     userdata = '''#!/usr/bin/env bash
-    curl -L -s https://gogs.informatik.hs-fulda.de/srieger/cloud-computing-msc-ai-examples/raw/master/faafo/contrib/install.sh | bash -s -- \
+    curl -L -s https://gogs.informatik.hs-fulda.de/srieger/cloud-computing-msc-ai-examples/raw/master/faafo/contrib/install-aws.sh | bash -s -- \
         -i faafo -r worker -e 'http://%(ip_controller)s' -m 'amqp://faafo:guest@%(ip_controller)s:5672/'
-    ''' % {'ip_controller': ip_controller}
+    ''' % {'ip_controller': private_ip_controller}
 
     print('Starting new app-worker-1 instance and wait until it is running...')
     instance_worker_1 = conn.create_node(name='app-worker-1',
@@ -253,7 +269,9 @@ def main():
                                          ex_userdata=userdata,
                                          ex_security_groups=[worker_security_group_name])
 
-    conn.wait_until_running(nodes=[instance_worker_1], timeout=120, ssh_interface='public_ips')
+    wait_until_running_result = conn.wait_until_running(nodes=[instance_worker_1], timeout=timeout, ssh_interface='public_ips')
+    instance_worker_1, node_addresses = wait_until_running_result[0]
+    print('Worker instance %s has private ip: %s ' % (instance_worker_1.id, instance_worker_1.private_ips[0]))
 
     ###########################################################################
     #
@@ -261,20 +279,22 @@ def main():
     #
     ###########################################################################
 
-    # AWS offers elastic ips, that have the same function as floating IPs in OpenStack. However elastic IPs cost money,
-    # and instances typically already have public IP in AWS, what a luxury ;)
-    print('Checking for unused Elastic IP...')
-    unused_elastic_ip = None
-    for elastic_ip in conn.ex_describe_all_addresses():
-        if not elastic_ip.instance_id:
-            unused_elastic_ip = elastic_ip
-            break
-
-    if not unused_elastic_ip:
-        print('Allocating new Elastic IP')
-        unused_elastic_ip = conn.ex_allocate_address()
-    conn.ex_associate_address_with_node(instance_worker_1, unused_elastic_ip)
-    print('The worker will be available for SSH at %s' % unused_elastic_ip.ip)
+    # AWS offers elastic ips, that have the same function as floating IPs in OpenStack. However, elastic IPs cost money,
+    # and instances typically already have public IP in AWS, what a luxury ;) so I commented out elastic IP creation to
+    # save your AWS academy budget
+    # print('Checking for unused Elastic IP...')
+    # unused_elastic_ip = None
+    # for elastic_ip in conn.ex_describe_all_addresses():
+    #     if not elastic_ip.instance_id:
+    #         unused_elastic_ip = elastic_ip
+    #         break
+    #
+    # if not unused_elastic_ip:
+    #     print('Allocating new Elastic IP')
+    #     unused_elastic_ip = conn.ex_allocate_address()
+    # conn.ex_associate_address_with_node(instance_worker_1, unused_elastic_ip)
+    # print('The worker will be available for SSH at %s' % unused_elastic_ip.ip)
+    #
 
     print('You can use ssh to login to the controller using your private key. After login, you can list available '
           'fractals using "faafo list". To request the generation of new fractals, you can use "faafo create". '