You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

305 lines
14 KiB

5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
  1. import configparser
  2. from os.path import expanduser
  3. from libcloud.compute.base import NodeImage
  4. from libcloud.compute.providers import get_driver
  5. from libcloud.compute.types import Provider
  6. home = expanduser("~")
  7. # reqs:
  8. # services: EC2 (nova, glance, neutron)
  9. # resources: 2 instances, 2 elastic ips (1 keypair, 2 security groups)
  10. # The image to look for and use for the started instance
  11. ubuntu_image_name = 'ubuntu/images/hvm-ssd/ubuntu-bionic-18.04-amd64-server-20210128'
  12. # TODO: 18.04, currently still needed for faafo, need to port faafo demo app to 20.04 or higher and python3...
  13. # The public key to be used for SSH connection, please make sure, that you have the corresponding private key
  14. #
  15. # id_rsa.pub should look like this (standard sshd pubkey format):
  16. # ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAw+J...F3w2mleybgT1w== user@HOSTNAME
  17. keypair_name = 'srieger-pub'
  18. pub_key_file = home + '/.ssh/id_rsa.pub'
  19. flavor_name = 't2.nano'
  20. # default region
  21. # region_name = 'eu-central-1'
  22. # region_name = 'ap-south-1'
  23. # AWS Academy Labs only allow us-east-1 see our AWS Academy Lab Guide, https://awsacademy.instructure.com/login/
  24. region_name = 'us-east-1'
  25. # starting instances in AWS Academy takes significantly longer compared to paid AWS accounts, allow ~ >2 minutes timeout
  26. timeout = 300
  27. def main():
  28. ###########################################################################
  29. #
  30. # get credentials
  31. #
  32. ###########################################################################
  33. # see AWS Academy Lab for Account Details
  34. # read credentials from file
  35. config = configparser.ConfigParser()
  36. config.read_file(open(home + '/.aws/credentials'))
  37. aws_access_key_id = config['default']['aws_access_key_id']
  38. aws_secret_access_key = config['default']['aws_secret_access_key']
  39. aws_session_token = config['default']['aws_session_token']
  40. # hard coded AWS credentials using vars
  41. # aws_access_key_id = "ASIAX..."
  42. # aws_secret_access_key = "WLxxXK+..."
  43. # aws_session_token = "FwoGZXIvYXdzEMb//////////wEaDE5rX.......0SleZ+L75I9iEri9LA4hovWul8HvexhCBK8.......................Ae/T+VkUbcQRtJEDwg+gYCABuk0JlSj5Wk7YA65r3BSNJXZFpkhbek6VBjvE/cEt5fKZEhENcdFxjAcAJLd6bOWi/oGXU5e3PX3mcXgm0oJpz6h3wqD1LvSDtw5GDwn0BHiF1Mu.......................cm/VukK5F"
  44. ###########################################################################
  45. #
  46. # create connection
  47. #
  48. ###########################################################################
  49. provider = get_driver(Provider.EC2)
  50. conn = provider(aws_access_key_id,
  51. aws_secret_access_key,
  52. token=aws_session_token,
  53. region=region_name)
  54. ###########################################################################
  55. #
  56. # get image, flavor, network for instance creation
  57. #
  58. ###########################################################################
  59. print("Search for AMI...")
  60. image = conn.list_images(ex_filters={"name": ubuntu_image_name})[0]
  61. print("Using image: %s" % image)
  62. # print("Fetching images (AMI) list from AWS region. This will take a lot of seconds (AWS has a very long list of "
  63. # "supported operating systems and versions)... please be patient...")
  64. # image = ''
  65. # for img in images:
  66. # # if img.name == ubuntu_image_name:
  67. # if img.extra['owner_alias'] == 'amazon':
  68. # print(img)
  69. # if img.id == ubuntu_image_name:
  70. # image = img
  71. # fetch/select the image referenced with ubuntu_image_name above
  72. # image = [i for i in images if i.name == ubuntu_image_name][0]
  73. # print(image)
  74. # select image directly to save time, as retrieving the image list takes several minutes now,
  75. # need to change ami id here if updated or for other regions, id is working for course in
  76. # summer term 2022, in region: us-east-1 and pointing to ubuntu 18.04 used in the instance wizard,
  77. # to update AMI id use the create instance wizard and copy amd64 image id for ubuntu 18.04 in the
  78. # desired region
  79. # image = NodeImage(id="ami-0e472ba40eb589f49",
  80. # name=ubuntu_image_name,
  81. # driver="hvm")
  82. flavors = conn.list_sizes()
  83. flavor = [s for s in flavors if s.id == flavor_name][0]
  84. print(flavor)
  85. # networks = conn.ex_list_networks()
  86. # network = ''
  87. # for net in networks:
  88. # if net.name == project_network:
  89. # network = net
  90. ###########################################################################
  91. #
  92. # create keypair dependency
  93. #
  94. ###########################################################################
  95. print('Checking for existing SSH key pair...')
  96. keypair_exists = False
  97. for keypair in conn.list_key_pairs():
  98. if keypair.name == keypair_name:
  99. keypair_exists = True
  100. if keypair_exists:
  101. print('Keypair ' + keypair_name + ' already exists. Skipping import.')
  102. else:
  103. print('adding keypair...')
  104. conn.import_key_pair_from_file(keypair_name, pub_key_file)
  105. for keypair in conn.list_key_pairs():
  106. print(keypair)
  107. ###########################################################################
  108. #
  109. # create security group dependency
  110. #
  111. ###########################################################################
  112. print('Checking for existing worker security group...')
  113. worker_security_group_exists = False
  114. worker_security_group_name = 'worker'
  115. for security_group in conn.ex_get_security_groups():
  116. if security_group.name == worker_security_group_name:
  117. worker_security_group_id = security_group.id
  118. worker_security_group_exists = True
  119. if worker_security_group_exists:
  120. print('Worker Security Group ' + worker_security_group_name + ' already exists. Skipping creation.')
  121. else:
  122. worker_security_group_result = conn.ex_create_security_group('worker', 'for services that run on a worker node')
  123. worker_security_group_id = worker_security_group_result['group_id']
  124. conn.ex_authorize_security_group_ingress(worker_security_group_id, 22, 22, cidr_ips=['0.0.0.0/0'],
  125. protocol='tcp')
  126. print('Checking for existing controller security group...')
  127. controller_security_group_exists = False
  128. controller_security_group_name = 'control'
  129. controller_security_group_id = ''
  130. for security_group in conn.ex_get_security_groups():
  131. if security_group.name == controller_security_group_name:
  132. controller_security_group_id = security_group.id
  133. controller_security_group_exists = True
  134. if controller_security_group_exists:
  135. print('Controller Security Group ' + controller_security_group_name + ' already exists. Skipping creation.')
  136. else:
  137. controller_security_group_result = conn.ex_create_security_group('control',
  138. 'for services that run on a control node')
  139. controller_security_group_id = controller_security_group_result['group_id']
  140. conn.ex_authorize_security_group_ingress(controller_security_group_id, 22, 22, cidr_ips=['0.0.0.0/0'],
  141. protocol='tcp')
  142. conn.ex_authorize_security_group_ingress(controller_security_group_id, 80, 80, cidr_ips=['0.0.0.0/0'],
  143. protocol='tcp')
  144. conn.ex_authorize_security_group_ingress(controller_security_group_id, 5672, 5672,
  145. group_pairs=[{'group_id': worker_security_group_id}], protocol='tcp')
  146. # for security_group in conn.ex_list_security_groups():
  147. # print(security_group)
  148. ###########################################################################
  149. #
  150. # create app-controller
  151. #
  152. ###########################################################################
  153. # https://git.openstack.org/cgit/openstack/faafo/plain/contrib/install.sh
  154. # is currently broken, hence the "rabbitctl" lines were added in the example
  155. # below, see also https://bugs.launchpad.net/faafo/+bug/1679710
  156. #
  157. # Thanks to Stefan Friedmann for finding this fix ;)
  158. userdata = '''#!/usr/bin/env bash
  159. curl -L -s https://gogs.informatik.hs-fulda.de/srieger/cloud-computing-msc-ai-examples/raw/master/faafo/contrib/install-aws.sh | bash -s -- \
  160. -i messaging -i faafo -r api
  161. rabbitmqctl add_user faafo guest
  162. rabbitmqctl set_user_tags faafo administrator
  163. rabbitmqctl set_permissions -p / faafo ".*" ".*" ".*"
  164. '''
  165. print('Starting new app-controller instance and wait until it is running (can take several minutes in free AWS'
  166. ' academy accounts compared to seconds when using a regular paid AWS account), timeout %i seconds...'
  167. % timeout)
  168. instance_controller_1 = conn.create_node(name='app-controller',
  169. image=image,
  170. size=flavor,
  171. ex_keyname=keypair_name,
  172. ex_userdata=userdata,
  173. ex_security_groups=[controller_security_group_name])
  174. wait_until_running_result = conn.wait_until_running(nodes=[instance_controller_1], timeout=timeout, ssh_interface='public_ips')
  175. instance_controller_1, node_addresses = wait_until_running_result[0]
  176. ###########################################################################
  177. #
  178. # assign app-controller elastic ip
  179. #
  180. ###########################################################################
  181. # AWS offers elastic ips, that have the same function as floating IPs in OpenStack. However, elastic IPs cost money,
  182. # and instances typically already have public IP in AWS, what a luxury ;) so I commented out elastic IP creation to
  183. # save your AWS academy budget
  184. # print('Checking for unused Elastic IP...')
  185. # unused_elastic_ip = None
  186. # for elastic_ip in conn.ex_describe_all_addresses():
  187. # if not elastic_ip.instance_id:
  188. # unused_elastic_ip = elastic_ip
  189. # break
  190. #
  191. # if not unused_elastic_ip:
  192. # print('Allocating new Elastic IP')
  193. # unused_elastic_ip = conn.ex_allocate_address()
  194. # conn.ex_associate_address_with_node(instance_controller_1, unused_elastic_ip)
  195. # print('Controller Application will be deployed to http://%s' % unused_elastic_ip.ip)
  196. ###########################################################################
  197. #
  198. # getting id and ip address of app-controller instance
  199. #
  200. ###########################################################################
  201. # instance_controller_1 = conn.list_nodes(ex_node_ids=instance_controller_1.id)
  202. public_ip_controller = instance_controller_1.public_ips[0]
  203. print('Controller Application %s will be reachable after cloud-init has run at http://%s, you can also connect'
  204. % (instance_controller_1.id, public_ip_controller) +
  205. ' via ssh ubuntu@%s' % public_ip_controller)
  206. # get private IP of instance to use in worker/service instances to connect to the controller
  207. private_ip_controller = instance_controller_1.private_ips[0]
  208. ###########################################################################
  209. #
  210. # create app-worker-1
  211. #
  212. ###########################################################################
  213. userdata = '''#!/usr/bin/env bash
  214. curl -L -s https://gogs.informatik.hs-fulda.de/srieger/cloud-computing-msc-ai-examples/raw/master/faafo/contrib/install-aws.sh | bash -s -- \
  215. -i faafo -r worker -e 'http://%(ip_controller)s' -m 'amqp://faafo:guest@%(ip_controller)s:5672/'
  216. ''' % {'ip_controller': private_ip_controller}
  217. print('Starting new app-worker-1 instance and wait until it is running...')
  218. instance_worker_1 = conn.create_node(name='app-worker-1',
  219. image=image,
  220. size=flavor,
  221. ex_keyname=keypair_name,
  222. ex_userdata=userdata,
  223. ex_security_groups=[worker_security_group_name])
  224. wait_until_running_result = conn.wait_until_running(nodes=[instance_worker_1], timeout=timeout, ssh_interface='public_ips')
  225. instance_worker_1, node_addresses = wait_until_running_result[0]
  226. print('Worker instance %s has private ip: %s ' % (instance_worker_1.id, instance_worker_1.private_ips[0]))
  227. ###########################################################################
  228. #
  229. # assign app-worker elastic ip
  230. #
  231. ###########################################################################
  232. # AWS offers elastic ips, that have the same function as floating IPs in OpenStack. However, elastic IPs cost money,
  233. # and instances typically already have public IP in AWS, what a luxury ;) so I commented out elastic IP creation to
  234. # save your AWS academy budget
  235. # print('Checking for unused Elastic IP...')
  236. # unused_elastic_ip = None
  237. # for elastic_ip in conn.ex_describe_all_addresses():
  238. # if not elastic_ip.instance_id:
  239. # unused_elastic_ip = elastic_ip
  240. # break
  241. #
  242. # if not unused_elastic_ip:
  243. # print('Allocating new Elastic IP')
  244. # unused_elastic_ip = conn.ex_allocate_address()
  245. # conn.ex_associate_address_with_node(instance_worker_1, unused_elastic_ip)
  246. # print('The worker will be available for SSH at %s' % unused_elastic_ip.ip)
  247. #
  248. print('You can use ssh to login to the controller using your private key. After login, you can list available '
  249. 'fractals using "faafo list". To request the generation of new fractals, you can use "faafo create". '
  250. 'You can also see other options to use the faafo example cloud service using "faafo -h".')
  251. if __name__ == '__main__':
  252. main()