diff --git a/faafo/.gitignore b/faafo/.gitignore new file mode 100644 index 0000000..a2a99f9 --- /dev/null +++ b/faafo/.gitignore @@ -0,0 +1,14 @@ +.tox +.vagrant +*.pyc +.venv +*.egg-info +*.egg +.eggs +.*.swp +build +*.log +*.png +AUTHORS +ChangeLog +dist diff --git a/faafo/CONTRIBUTING.rst b/faafo/CONTRIBUTING.rst new file mode 100644 index 0000000..73cc4a8 --- /dev/null +++ b/faafo/CONTRIBUTING.rst @@ -0,0 +1,16 @@ +If you would like to contribute to the development of OpenStack, +you must follow the steps documented at: + + http://docs.openstack.org/infra/manual/developers.html#getting-started + +Once those steps have been completed, changes to OpenStack +should be submitted for review via the Gerrit tool, following +the workflow documented at: + + http://docs.openstack.org/infra/manual/developers.html#development-workflow + +Pull requests submitted through GitHub will be ignored. + +Bugs should be filed on Launchpad, not GitHub: + + https://bugs.launchpad.net/faafo diff --git a/faafo/LICENSE b/faafo/LICENSE new file mode 100644 index 0000000..e06d208 --- /dev/null +++ b/faafo/LICENSE @@ -0,0 +1,202 @@ +Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright {yyyy} {name of copyright owner} + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + diff --git a/faafo/README.rst b/faafo/README.rst new file mode 100644 index 0000000..ab13310 --- /dev/null +++ b/faafo/README.rst @@ -0,0 +1,30 @@ +Local copy of the former faafo OpenStack Tutorial from developer.openstack.org + +======================== +Team and repository tags +======================== + +.. image:: http://governance.openstack.org/badges/faafo.svg + :target: http://governance.openstack.org/reference/tags/index.html + +.. Change things from this point on + +=========================================== +First App Application for OpenStack (faafo) +=========================================== + +Developer documentation, the source of which is in ``doc/source/``, is +published at: + + http://docs.openstack.org/developer/faafo/ + +Bugs and feature requests are tracked on Launchpad at: + + https://bugs.launchpad.net/faafo + +Contributors are encouraged to join IRC (``#openstackfirstapp`` on freenode): + + https://wiki.openstack.org/wiki/IRC + +For information on contributing to the First App Application for OpenStack, +see ``CONTRIBUTING.rst``. diff --git a/faafo/Vagrantfile b/faafo/Vagrantfile new file mode 100644 index 0000000..8e5a603 --- /dev/null +++ b/faafo/Vagrantfile @@ -0,0 +1,49 @@ +# -*- mode: ruby -*- +# vi: set ft=ruby : + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +Vagrant.configure(2) do |config| + config.vm.box = 'ubuntu/trusty64' + config.vm.provider 'virtualbox' do |vb| + vb.memory = 1024 + vb.cpus = 1 + end + config.ssh.shell = 'bash -c "BASH_ENV=/etc/profile exec bash"' + config.cache.scope = :box if Vagrant.has_plugin?('vagrant-cachier') + config.vm.provision "shell", + inline: "apt-get update && apt-get upgrade -y" + + config.vm.define "services", primary: true do |node| + node.vm.hostname= "services" + node.vm.network :private_network, ip: '10.0.88.10' + node.vm.provision "shell", + inline: "/vagrant/contrib/install.sh -i messaging -i database" + node.vm.network 'forwarded_port', guest: 15_672, host: 15_672 + end + + config.vm.define "api" do |node| + node.vm.hostname= "api" + node.vm.network :private_network, ip: '10.0.88.20' + node.vm.provision "shell", + inline: "/vagrant/contrib/install.sh -i faafo -r api -d 'mysql://faafo:password@10.0.88.10:3306/faafo' -m 'amqp://guest:guest@10.0.88.10:5672/'" + node.vm.network 'forwarded_port', guest: 80, host: 1080 + end + + config.vm.define "worker" do |node| + node.vm.hostname= "worker" + node.vm.network :private_network, ip: '10.0.88.30' + node.vm.provision "shell", + inline: "/vagrant/contrib/install.sh -i faafo -r worker -m 'amqp://guest:guest@10.0.88.10:5672/' -e 'http://10.0.88.20'" + end +end diff --git a/faafo/bin/faafo b/faafo/bin/faafo new file mode 100644 index 0000000..a12770d --- /dev/null +++ b/faafo/bin/faafo @@ -0,0 +1,267 @@ +#!/usr/bin/env python + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import copy +import json +import random +import uuid + +from oslo_config import cfg +from oslo_log import log +from prettytable import PrettyTable +import requests + +from faafo import version + + +LOG = log.getLogger('faafo.client') +CONF = cfg.CONF + + +def get_random_task(): + random.seed() + + if CONF.command.width: + width = int(CONF.command.width) + else: + width = random.randint(int(CONF.command.min_width), + int(CONF.command.max_width)) + + if CONF.command.height: + height = int(CONF.command.height) + else: + height = random.randint(int(CONF.command.min_height), + int(CONF.command.max_height)) + + if CONF.command.iterations: + iterations = int(CONF.command.iterations) + else: + iterations = random.randint(int(CONF.command.min_iterations), + int(CONF.command.max_iterations)) + + if CONF.command.xa: + xa = float(CONF.command.xa) + else: + xa = random.uniform(float(CONF.command.min_xa), + float(CONF.command.max_xa)) + + if CONF.command.xb: + xb = float(CONF.command.xb) + else: + xb = random.uniform(float(CONF.command.min_xb), + float(CONF.command.max_xb)) + + if CONF.command.ya: + ya = float(CONF.command.ya) + else: + ya = random.uniform(float(CONF.command.min_ya), + float(CONF.command.max_ya)) + + if CONF.command.yb: + yb = float(CONF.command.yb) + else: + yb = random.uniform(float(CONF.command.min_yb), + float(CONF.command.max_yb)) + + task = { + 'uuid': str(uuid.uuid4()), + 'width': width, + 'height': height, + 'iterations': iterations, 'xa': xa, + 'xb': xb, + 'ya': ya, + 'yb': yb + } + + return task + + +def do_get_fractal(): + LOG.error("command 'download' not yet implemented") + + +def do_show_fractal(): + LOG.info("showing fractal %s" % CONF.command.uuid) + result = requests.get("%s/v1/fractal/%s" % + (CONF.endpoint_url, CONF.command.uuid)) + if result.status_code == 200: + data = json.loads(result.text) + output = PrettyTable(["Parameter", "Value"]) + output.align["Parameter"] = "l" + output.align["Value"] = "l" + output.add_row(["uuid", data['uuid']]) + output.add_row(["duration", "%f seconds" % data['duration']]) + output.add_row(["dimensions", "%d x %d pixels" % + (data['width'], data['height'])]) + output.add_row(["iterations", data['iterations']]) + output.add_row(["xa", data['xa']]) + output.add_row(["xb", data['xb']]) + output.add_row(["ya", data['ya']]) + output.add_row(["yb", data['yb']]) + output.add_row(["size", "%d bytes" % data['size']]) + output.add_row(["checksum", data['checksum']]) + output.add_row(["generated_by", data['generated_by']]) + print(output) + else: + LOG.error("fractal '%s' not found" % CONF.command.uuid) + + +def do_list_fractals(): + LOG.info("listing all fractals") + + fractals = get_fractals() + output = PrettyTable(["UUID", "Dimensions", "Filesize"]) + for fractal in fractals: + output.add_row([ + fractal["uuid"], + "%d x %d pixels" % (fractal["width"], fractal["height"]), + "%d bytes" % (fractal["size"] or 0), + ]) + print(output) + + +def get_fractals(page=1): + result = requests.get("%s/v1/fractal?page=%d" % + (CONF.endpoint_url, page)) + + fractals = [] + if result.status_code == 200: + data = json.loads(result.text) + if page < data['total_pages']: + fractals = data['objects'] + get_fractals(page + 1) + else: + return data['objects'] + + return fractals + + +def do_delete_fractal(): + LOG.info("deleting fractal %s" % CONF.command.uuid) + result = requests.delete("%s/v1/fractal/%s" % + (CONF.endpoint_url, CONF.command.uuid)) + LOG.debug("result: %s" %result) + + +def do_create_fractal(): + random.seed() + if CONF.command.tasks: + number = int(CONF.command.tasks) + else: + number = random.randint(int(CONF.command.min_tasks), + int(CONF.command.max_tasks)) + LOG.info("generating %d task(s)" % number) + for i in xrange(0, number): + task = get_random_task() + LOG.debug("created task %s" % task) + # NOTE(berendt): only necessary when using requests < 2.4.2 + headers = {'Content-type': 'application/json', + 'Accept': 'text/plain'} + requests.post("%s/v1/fractal" % CONF.endpoint_url, + json.dumps(task), headers=headers) + + +def add_command_parsers(subparsers): + parser = subparsers.add_parser('create') + parser.set_defaults(func=do_create_fractal) + parser.add_argument("--height", default=None, + help="The height of the generate image.") + parser.add_argument("--min-height", default=256, + help="The minimum height of the generate image.") + parser.add_argument("--max-height", default=1024, + help="The maximum height of the generate image.") + parser.add_argument("--width", default=None, + help="The width of the generated image.") + parser.add_argument("--min-width", default=256, + help="The minimum width of the generated image.") + parser.add_argument("--max-width", default=1024, + help="The maximum width of the generated image.") + parser.add_argument("--iterations", default=None, + help="The number of iterations.") + parser.add_argument("--min-iterations", default=128, + help="The minimum number of iterations.") + parser.add_argument("--max-iterations", default=512, + help="The maximum number of iterations.") + parser.add_argument("--tasks", default=None, + help="The number of generated fractals.") + parser.add_argument("--min-tasks", default=1, + help="The minimum number of generated fractals.") + parser.add_argument("--max-tasks", default=10, + help="The maximum number of generated fractals.") + parser.add_argument("--xa", default=None, + help="The value for the parameter 'xa'.") + parser.add_argument("--min-xa", default=-1.0, + help="The minimum value for the parameter 'xa'.") + parser.add_argument("--max-xa", default=-4.0, + help="The maximum value for the parameter 'xa'.") + parser.add_argument("--xb", default=None, + help="The value for the parameter 'xb'.") + parser.add_argument("--min-xb", default=1.0, + help="The minimum value for the parameter 'xb'.") + parser.add_argument("--max-xb", default=4.0, + help="The maximum value for the parameter 'xb'.") + parser.add_argument("--ya", default=None, + help="The value for the parameter 'ya'.") + parser.add_argument("--min-ya", default=-0.5, + help="The minimum value for the parameter 'ya'.") + parser.add_argument("--max-ya", default=-3, + help="The maximum value for the parameter 'ya'.") + parser.add_argument("--yb", default=None, + help="The value for the parameter 'yb'.") + parser.add_argument("--min-yb", default=0.5, + help="The minimum value for the parameter 'yb'.") + parser.add_argument("--max-yb", default=3, + help="The maximum value for the parameter 'yb'.") + + parser = subparsers.add_parser('delete') + parser.set_defaults(func=do_delete_fractal) + parser.add_argument("uuid", help="Fractal to delete.") + + parser = subparsers.add_parser('show') + parser.set_defaults(func=do_show_fractal) + parser.add_argument("uuid", help="Fractal to show.") + + parser = subparsers.add_parser('get') + parser.set_defaults(func=do_get_fractal) + parser.add_argument("uuid", help="Fractal to download.") + + parser = subparsers.add_parser('list') + parser.set_defaults(func=do_list_fractals) + + +client_commands = cfg.SubCommandOpt('command', title='Commands', + help='Show available commands.', + handler=add_command_parsers) + +CONF.register_cli_opts([client_commands]) + +client_cli_opts = [ + cfg.StrOpt('endpoint-url', + default='http://localhost', + help='API connection URL') +] + +CONF.register_cli_opts(client_cli_opts) + + +if __name__ == '__main__': + log.register_options(CONF) + log.set_defaults() + + CONF(project='client', prog='faafo-client', + version=version.version_info.version_string()) + + log.setup(CONF, 'client', + version=version.version_info.version_string()) + + CONF.command.func() diff --git a/faafo/bin/faafo-worker b/faafo/bin/faafo-worker new file mode 100644 index 0000000..67daf0b --- /dev/null +++ b/faafo/bin/faafo-worker @@ -0,0 +1,52 @@ +#!/usr/bin/env python + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import os +import sys + +import kombu +from oslo_config import cfg +from oslo_log import log + +from faafo.worker import service as worker +from faafo import version + +LOG = log.getLogger('faafo.worker') +CONF = cfg.CONF + +# If ../faafo/__init__.py exists, add ../ to Python search path, so that +# it will override what happens to be installed in /usr/(local/)lib/python... +possible_topdir = os.path.normpath(os.path.join(os.path.abspath(sys.argv[0]), + os.pardir, + os.pardir)) +if os.path.exists(os.path.join(possible_topdir, 'faafo', '__init__.py')): + sys.path.insert(0, possible_topdir) + +if __name__ == '__main__': + log.register_options(CONF) + log.set_defaults() + + CONF(project='worker', prog='faafo-worker', + default_config_files=['/etc/faafo/faafo.conf'], + version=version.version_info.version_string()) + + log.setup(CONF, 'worker', + version=version.version_info.version_string()) + + connection = kombu.Connection(CONF.transport_url) + server = worker.Worker(connection) + try: + server.run() + except KeyboardInterrupt: + LOG.info("Caught keyboard interrupt. Exiting.") diff --git a/faafo/contrib/install.sh b/faafo/contrib/install.sh new file mode 100644 index 0000000..63e2d37 --- /dev/null +++ b/faafo/contrib/install.sh @@ -0,0 +1,198 @@ +#!/bin/bash + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +if [[ -e /etc/os-release ]]; then + + # NOTE(berendt): support for CentOS/RHEL/openSUSE/SLES will be added in the future + + source /etc/os-release + + INSTALL_DATABASE=0 + INSTALL_FAAFO=0 + INSTALL_MESSAGING=0 + RUN_API=0 + RUN_DEMO=0 + RUN_WORKER=0 + URL_DATABASE='sqlite:////tmp/sqlite.db' + URL_ENDPOINT='http://127.0.0.1' + URL_MESSAGING='amqp://guest:guest@localhost:5672/' + + while getopts e:m:d:i:r: FLAG; do + case $FLAG in + i) + case $OPTARG in + messaging) + INSTALL_MESSAGING=1 + ;; + database) + INSTALL_DATABASE=1 + ;; + faafo) + INSTALL_FAAFO=1 + ;; + esac + ;; + r) + case $OPTARG in + demo) + RUN_DEMO=1 + ;; + api) + RUN_API=1 + ;; + worker) + RUN_WORKER=1 + ;; + esac + ;; + e) + URL_ENDPOINT=$OPTARG + ;; + + m) + URL_MESSAGING=$OPTARG + ;; + + d) + URL_DATABASE=$OPTARG + ;; + + *) + echo "error: unknown option $FLAG" + exit 1 + ;; + esac + done + + if [[ $ID = 'ubuntu' || $ID = 'debian' ]]; then + sudo apt-get update + elif [[ $ID = 'fedora' ]]; then + sudo dnf update -y + fi + + if [[ $INSTALL_DATABASE -eq 1 ]]; then + if [[ $ID = 'ubuntu' || $ID = 'debian' ]]; then + sudo DEBIAN_FRONTEND=noninteractive apt-get install -y mysql-server python-mysqldb + sudo sed -i -e "/bind-address/d" /etc/mysql/my.cnf + sudo service mysql restart + elif [[ $ID = 'fedora' ]]; then + sudo dnf install -y mariadb-server python-mysql + printf "[mysqld]\nbind-address = 127.0.0.1\n" | sudo tee /etc/my.cnf.d/faafo.conf + sudo systemctl enable mariadb + sudo systemctl start mariadb + else + echo "error: distribution $ID not supported" + exit 1 + fi + sudo mysqladmin password password + sudo mysql -uroot -ppassword mysql -e "CREATE DATABASE IF NOT EXISTS faafo; GRANT ALL PRIVILEGES ON faafo.* TO 'faafo'@'%' IDENTIFIED BY 'password';" + URL_DATABASE='mysql://root:password@localhost/faafo' + fi + + if [[ $INSTALL_MESSAGING -eq 1 ]]; then + if [[ $ID = 'ubuntu' || $ID = 'debian' ]]; then + sudo apt-get install -y rabbitmq-server + elif [[ $ID = 'fedora' ]]; then + sudo dnf install -y rabbitmq-server + sudo systemctl enable rabbitmq-server + sudo systemctl start rabbitmq-server + else + echo "error: distribution $ID not supported" + exit 1 + fi + fi + + if [[ $INSTALL_FAAFO -eq 1 ]]; then + if [[ $ID = 'ubuntu' || $ID = 'debian' ]]; then + sudo apt-get install -y python-dev python-pip supervisor git zlib1g-dev libmysqlclient-dev python-mysqldb + # Following is needed because of + # https://bugs.launchpad.net/ubuntu/+source/supervisor/+bug/1594740 + if [ $(lsb_release --short --codename) = xenial ]; then + # Make sure the daemon is enabled. + if ! systemctl --quiet is-enabled supervisor; then + systemctl enable supervisor + fi + # Make sure the daemon is started. + if ! systemctl --quiet is-active supervisor; then + systemctl start supervisor + fi + fi + elif [[ $ID = 'fedora' ]]; then + sudo dnf install -y python-devel python-pip supervisor git zlib-devel mariadb-devel gcc which python-mysql + sudo systemctl enable supervisord + sudo systemctl start supervisord + #elif [[ $ID = 'opensuse' || $ID = 'sles' ]]; then + # sudo zypper install -y python-devel python-pip + else + echo "error: distribution $ID not supported" + exit 1 + fi + + git clone https://git.openstack.org/openstack/faafo + cd faafo + # following line required by bug 1636150 + sudo pip install --upgrade pbr + sudo pip install -r requirements.txt + sudo python setup.py install + + sudo sed -i -e "s#transport_url = .*#transport_url = $URL_MESSAGING#" /etc/faafo/faafo.conf + sudo sed -i -e "s#database_url = .*#database_url = $URL_DATABASE#" /etc/faafo/faafo.conf + sudo sed -i -e "s#endpoint_url = .*#endpoint_url = $URL_ENDPOINT#" /etc/faafo/faafo.conf + fi + + + if [[ $RUN_API -eq 1 ]]; then + faafo_api=" +[program:faafo_api] +command=$(which faafo-api) +priority=10" + + if [[ $ID = 'ubuntu' || $ID = 'debian' ]]; then + echo "$faafo_api" | sudo tee -a /etc/supervisor/conf.d/faafo.conf + elif [[ $ID = 'fedora' ]]; then + echo "$faafo_api" | sudo tee -a /etc/supervisord.d/faafo.ini + else + echo "error: distribution $ID not supported" + exit 1 + fi + fi + + if [[ $RUN_WORKER -eq 1 ]]; then + faafo_worker=" +[program:faafo_worker] +command=$(which faafo-worker) +priority=20" + + if [[ $ID = 'ubuntu' || $ID = 'debian' ]]; then + echo "$faafo_worker" | sudo tee -a /etc/supervisor/conf.d/faafo.conf + elif [[ $ID = 'fedora' ]]; then + echo "$faafo_worker" | sudo tee -a /etc/supervisord.d/faafo.ini + else + echo "error: distribution $ID not supported" + exit 1 + fi + fi + + if [[ $RUN_WORKER -eq 1 || $RUN_API -eq 1 ]]; then + sudo supervisorctl reload + sleep 5 + fi + + if [[ $RUN_DEMO -eq 1 && $RUN_API -eq 1 ]]; then + faafo --endpoint-url $URL_ENDPOINT --debug create + fi + +else + exit 1 +fi diff --git a/faafo/contrib/test_api.py b/faafo/contrib/test_api.py new file mode 100644 index 0000000..559ac3d --- /dev/null +++ b/faafo/contrib/test_api.py @@ -0,0 +1,56 @@ +#!/usr/bin/env python + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import json +import requests + +url = 'http://127.0.0.1/api/fractal' +headers = {'Content-Type': 'application/json'} + +uuid = '13bf15a8-9f6c-4d59-956f-7d20f7484687' +data = { + 'uuid': uuid, + 'width': 100, + 'height': 100, + 'iterations': 10, + 'xa': 1.0, + 'xb': -1.0, + 'ya': 1.0, + 'yb': -1.0, +} +response = requests.post(url, data=json.dumps(data), headers=headers) +assert response.status_code == 201 + +response = requests.get(url, headers=headers) +assert response.status_code == 200 +print(response.json()) + +response = requests.get(url + '/' + uuid, headers=headers) +assert response.status_code == 200 +print(response.json()) + +data = { + 'checksum': 'c6fef4ef13a577066c2281b53c82ce2c7e94e', + 'duration': 10.12 +} +response = requests.put(url + '/' + uuid, data=json.dumps(data), + headers=headers) +assert response.status_code == 200 + +response = requests.get(url + '/' + uuid, headers=headers) +assert response.status_code == 200 +print(response.json()) + +response = requests.delete(url + '/' + uuid, headers=headers) +assert response.status_code == 204 diff --git a/faafo/doc/source/conf.py b/faafo/doc/source/conf.py new file mode 100644 index 0000000..d78328c --- /dev/null +++ b/faafo/doc/source/conf.py @@ -0,0 +1,17 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +copyright = u'2015, OpenStack contributors' +master_doc = 'index' +project = u'First App Application for OpenStack' +source_suffix = '.rst' diff --git a/faafo/doc/source/development.rst b/faafo/doc/source/development.rst new file mode 100644 index 0000000..c8fd05a --- /dev/null +++ b/faafo/doc/source/development.rst @@ -0,0 +1,68 @@ +Development +=========== + +Vagrant environment +------------------- + +The `Vagrant `_ environment and the `Ansible `_ +playbook is used only for local tests and development of the application. + +The installation of Vagrant is described at https://docs.vagrantup.com/v2/installation/index.html. + +To speedup the provisioning you can install the Vagrant plugin `vagrant-cachier `_. + +.. code:: + + $ vagrant plugin install vagrant-cachier + +Bootstrap the Vagrant environment. + +.. code:: + + $ vagrant up + +Now it is possible to login with SSH. + +.. code:: + + $ vagrant ssh + +Open a new screen or tmux session. Aftwards run the api, worker, and producer +services in the foreground, each service in a separate window. + +* :code:`faafo-api` +* :code:`faafo-worker` +* :code:`faafo-producer` + +RabbitMQ server +~~~~~~~~~~~~~~~ + +The webinterface of the RabbitMQ server is reachable on TCP port :code:`15672`. The login is +possible with the user :code:`guest` and the password :code:`guest`. + +MySQL server +~~~~~~~~~~~~ + +The password of the user :code:`root` is :code:`secretsecret`. The password of the user :code:`faafo` +for the database :code:`faafo` is also :code:`secretsecret`. + +Virtual environment +------------------- + +Create a new virtual environment, install all required dependencies and +the application itself. + +.. code:: + + $ virtualenv .venv + $ source .venv/bin/activate + $ pip install -r requirements.txt + $ python setup.py install + +Open a new screen or tmux session. Aftwards run the api and worker +services in the foreground, each service in a separate window. + +.. code:: + + $ source .venv/bin/activate; faafo-api + $ source .venv/bin/activate; faafo-worker diff --git a/faafo/doc/source/images/diagram.dot b/faafo/doc/source/images/diagram.dot new file mode 100644 index 0000000..f1689a9 --- /dev/null +++ b/faafo/doc/source/images/diagram.dot @@ -0,0 +1,15 @@ +digraph { + API -> Database [color=green]; + API -> Database [color=orange]; + Database -> API [color=red]; + API -> Webinterface [color=red]; + Producer -> API [color=orange]; + Producer -> API [color=green]; + Producer -> "Queue Service" [color=orange]; + "Queue Service" -> Worker [color=orange]; + Worker -> "Image File" [color=blue]; + Worker -> "Queue Service" [color=green]; + "Queue Service" -> Producer [color=green]; + "Image File" -> "Storage Backend" [color=blue]; + "Storage Backend" -> Webinterface [color=red]; +} diff --git a/faafo/doc/source/images/diagram.png b/faafo/doc/source/images/diagram.png new file mode 100644 index 0000000..ea27612 Binary files /dev/null and b/faafo/doc/source/images/diagram.png differ diff --git a/faafo/doc/source/images/dot2png.sh b/faafo/doc/source/images/dot2png.sh new file mode 100644 index 0000000..8ae913f --- /dev/null +++ b/faafo/doc/source/images/dot2png.sh @@ -0,0 +1,3 @@ +#!/bin/sh + +dot -T png -o diagram.png diagram.dot diff --git a/faafo/doc/source/images/example.png b/faafo/doc/source/images/example.png new file mode 100644 index 0000000..e840fc4 Binary files /dev/null and b/faafo/doc/source/images/example.png differ diff --git a/faafo/doc/source/images/screenshot_webinterface.png b/faafo/doc/source/images/screenshot_webinterface.png new file mode 100644 index 0000000..053d241 Binary files /dev/null and b/faafo/doc/source/images/screenshot_webinterface.png differ diff --git a/faafo/doc/source/implementation.rst b/faafo/doc/source/implementation.rst new file mode 100644 index 0000000..d95013d --- /dev/null +++ b/faafo/doc/source/implementation.rst @@ -0,0 +1,11 @@ +Implementation +============== + +Frameworks +---------- + +* http://flask.pocoo.org/ +* http://python-requests.org +* http://www.sqlalchemy.org/ +* https://github.com/celery/kombu +* https://pillow.readthedocs.org/ diff --git a/faafo/doc/source/index.rst b/faafo/doc/source/index.rst new file mode 100644 index 0000000..0204278 --- /dev/null +++ b/faafo/doc/source/index.rst @@ -0,0 +1,14 @@ +First App Application for OpenStack +=================================== + +Contents: + +.. toctree:: + :maxdepth: 2 + + workflow + implementation + installation + usage + development + references diff --git a/faafo/doc/source/installation.rst b/faafo/doc/source/installation.rst new file mode 100644 index 0000000..4a44fba --- /dev/null +++ b/faafo/doc/source/installation.rst @@ -0,0 +1,8 @@ +Installation +============ + +To install the ``First App Application for OpenStack`` run the following command. + +.. code:: + + pip install faafo diff --git a/faafo/doc/source/references.rst b/faafo/doc/source/references.rst new file mode 100644 index 0000000..6748135 --- /dev/null +++ b/faafo/doc/source/references.rst @@ -0,0 +1,6 @@ +References +========== + +* http://en.wikipedia.org/wiki/Julia_set +* http://en.wikipedia.org/wiki/Mandelbrot_set +* http://code.activestate.com/recipes/577120-julia-fractals/ diff --git a/faafo/doc/source/usage.rst b/faafo/doc/source/usage.rst new file mode 100644 index 0000000..4f79d05 --- /dev/null +++ b/faafo/doc/source/usage.rst @@ -0,0 +1,42 @@ +Usage +===== + +Example image +------------- + +.. image:: images/example.png + + +Example outputs +--------------- + +Producer service +~~~~~~~~~~~~~~~~ + +.. code:: + + 2015-03-25 23:01:29.308 22526 INFO faafo.producer [-] generating 1 task(s) + 2015-03-25 23:01:29.344 22526 INFO faafo.producer [-] generated task: {'width': 510, 'yb': 2.478654026560605, 'uuid': '212e8c23-e67f-4bd3-86e1-5a5e811ee2f4', 'iterations': 281, 'xb': 1.1428457603077387, 'xa': -3.3528957195683087, 'ya': -2.1341119130263717, 'height': 278} + 2015-03-25 23:01:30.295 22526 INFO faafo.producer [-] task 212e8c23-e67f-4bd3-86e1-5a5e811ee2f4 processed: {u'duration': 0.8725259304046631, u'checksum': u'b22d975c4f9dc77df5db96ce6264a4990d865dd8f800aba2ac03a065c2f09b1e', u'uuid': u'212e8c23-e67f-4bd3-86e1-5a5e811ee2f4'} + +Worker service +~~~~~~~~~~~~~~ + +.. code:: + + 2015-03-25 23:01:29.378 22518 INFO faafo.worker [-] processing task 212e8c23-e67f-4bd3-86e1-5a5e811ee2f4 + 2015-03-25 23:01:30.251 22518 INFO faafo.worker [-] task 212e8c23-e67f-4bd3-86e1-5a5e811ee2f4 processed in 0.872526 seconds + 2015-03-25 23:01:30.268 22518 INFO faafo.worker [-] saved result of task 212e8c23-e67f-4bd3-86e1-5a5e811ee2f4 to file /home/vagrant/212e8c23-e67f-4bd3-86e1-5a5e811ee2f4.png + + +API Service +~~~~~~~~~~~ +.. code:: + + 2015-03-25 23:01:29.342 22511 INFO werkzeug [-] 127.0.0.1 - - [25/Mar/2015 23:01:29] "POST /api/fractal HTTP/1.1" 201 - + 2015-03-25 23:01:30.317 22511 INFO werkzeug [-] 127.0.0.1 - - [25/Mar/2015 23:01:30] "PUT /api/fractal/212e8c23-e67f-4bd3-86e1-5a5e811ee2f4 HTTP/1.1" 200 - + +Example webinterface view +------------------------- + +.. image:: images/screenshot_webinterface.png diff --git a/faafo/doc/source/workflow.rst b/faafo/doc/source/workflow.rst new file mode 100644 index 0000000..9f642dc --- /dev/null +++ b/faafo/doc/source/workflow.rst @@ -0,0 +1,4 @@ +Workflow +-------- + +.. image:: images/diagram.png diff --git a/faafo/etc/faafo.conf b/faafo/etc/faafo.conf new file mode 100644 index 0000000..fd8dbff --- /dev/null +++ b/faafo/etc/faafo.conf @@ -0,0 +1,110 @@ +[DEFAULT] + +# +# From faafo.api +# + +# Listen address. (string value) +#listen_address = 0.0.0.0 + +# Bind port. (integer value) +#bind_port = 80 + +# Database connection URL. (string value) +database_url = sqlite:////tmp/sqlite.db + +# +# From faafo.queues +# + +# AMQP connection URL. (string value) +transport_url = amqp://guest:guest@localhost:5672// + +# +# From faafo.worker +# + +# API connection URL (string value) +endpoint_url = http://localhost + +# +# From oslo.log +# + +# Print debugging output (set logging level to DEBUG instead of +# default WARNING level). (boolean value) +#debug = false + +# The name of a logging configuration file. This file is appended to +# any existing logging configuration files. For details about logging +# configuration files, see the Python logging module documentation. +# (string value) +# Deprecated group/name - [DEFAULT]/log_config +#log_config_append = + +# DEPRECATED. A logging.Formatter log message format string which may +# use any of the available logging.LogRecord attributes. This option +# is deprecated. Please use logging_context_format_string and +# logging_default_format_string instead. (string value) +#log_format = + +# Format string for %%(asctime)s in log records. Default: %(default)s +# . (string value) +#log_date_format = %Y-%m-%d %H:%M:%S + +# (Optional) Name of log file to output to. If no default is set, +# logging will go to stdout. (string value) +# Deprecated group/name - [DEFAULT]/logfile +#log_file = + +# (Optional) The base directory used for relative --log-file paths. +# (string value) +# Deprecated group/name - [DEFAULT]/logdir +#log_dir = + +# Use syslog for logging. Existing syslog format is DEPRECATED during +# I, and will change in J to honor RFC5424. (boolean value) +#use_syslog = false + +# (Optional) Enables or disables syslog rfc5424 format for logging. If +# enabled, prefixes the MSG part of the syslog message with APP-NAME +# (RFC5424). The format without the APP-NAME is deprecated in I, and +# will be removed in J. (boolean value) +#use_syslog_rfc_format = false + +# Syslog facility to receive log lines. (string value) +#syslog_log_facility = LOG_USER + +# Log output to standard error. (boolean value) +#use_stderr = true + +# Format string to use for log messages with context. (string value) +#logging_context_format_string = %(asctime)s.%(msecs)03d %(process)d %(levelname)s %(name)s [%(request_id)s %(user_identity)s] %(instance)s%(message)s + +# Format string to use for log messages without context. (string +# value) +#logging_default_format_string = %(asctime)s.%(msecs)03d %(process)d %(levelname)s %(name)s [-] %(instance)s%(message)s + +# Data to append to log format when level is DEBUG. (string value) +#logging_debug_format_suffix = %(funcName)s %(pathname)s:%(lineno)d + +# Prefix each line of exception output with this format. (string +# value) +#logging_exception_prefix = %(asctime)s.%(msecs)03d %(process)d TRACE %(name)s %(instance)s + +# List of logger=LEVEL pairs. (list value) +#default_log_levels = amqp=WARN,amqplib=WARN,boto=WARN,qpid=WARN,sqlalchemy=WARN,suds=INFO,oslo.messaging=INFO,iso8601=WARN,requests.packages.urllib3.connectionpool=WARN,urllib3.connectionpool=WARN,websocket=WARN,requests.packages.urllib3.util.retry=WARN,urllib3.util.retry=WARN,keystonemiddleware=WARN,routes.middleware=WARN,stevedore=WARN + +# Enables or disables publication of error events. (boolean value) +#publish_errors = false + +# Enables or disables fatal status of deprecations. (boolean value) +#fatal_deprecations = false + +# The format for an instance that is passed with the log message. +# (string value) +#instance_format = "[instance: %(uuid)s] " + +# The format for an instance UUID that is passed with the log message. +# (string value) +#instance_uuid_format = "[instance: %(uuid)s] " diff --git a/faafo/etc/faafo.conf.sample b/faafo/etc/faafo.conf.sample new file mode 100644 index 0000000..b6ebfce --- /dev/null +++ b/faafo/etc/faafo.conf.sample @@ -0,0 +1,125 @@ +[DEFAULT] + +# +# From faafo.api +# + +# Listen address. (string value) +#listen_address = 0.0.0.0 + +# Bind port. (integer value) +#bind_port = 80 + +# Database connection URL. (string value) +#database_url = sqlite:////tmp/sqlite.db + +# +# From faafo.queues +# + +# AMQP connection URL. (string value) +#transport_url = amqp://guest:guest@localhost:5672// + +# +# From faafo.worker +# + +# API connection URL (string value) +#endpoint_url = http://localhost + +# +# From oslo.log +# + +# If set to true, the logging level will be set to DEBUG instead of +# the default INFO level. (boolean value) +#debug = false + +# If set to false, the logging level will be set to WARNING instead of +# the default INFO level. (boolean value) +# This option is deprecated for removal. +# Its value may be silently ignored in the future. +#verbose = true + +# The name of a logging configuration file. This file is appended to +# any existing logging configuration files. For details about logging +# configuration files, see the Python logging module documentation. +# Note that when logging configuration files are used then all logging +# configuration is set in the configuration file and other logging +# configuration options are ignored (for example, +# logging_context_format_string). (string value) +# Deprecated group/name - [DEFAULT]/log_config +#log_config_append = + +# Defines the format string for %%(asctime)s in log records. Default: +# %(default)s . This option is ignored if log_config_append is set. +# (string value) +#log_date_format = %Y-%m-%d %H:%M:%S + +# (Optional) Name of log file to send logging output to. If no default +# is set, logging will go to stderr as defined by use_stderr. This +# option is ignored if log_config_append is set. (string value) +# Deprecated group/name - [DEFAULT]/logfile +#log_file = + +# (Optional) The base directory used for relative log_file paths. +# This option is ignored if log_config_append is set. (string value) +# Deprecated group/name - [DEFAULT]/logdir +#log_dir = + +# Uses logging handler designed to watch file system. When log file is +# moved or removed this handler will open a new log file with +# specified path instantaneously. It makes sense only if log_file +# option is specified and Linux platform is used. This option is +# ignored if log_config_append is set. (boolean value) +#watch_log_file = false + +# Use syslog for logging. Existing syslog format is DEPRECATED and +# will be changed later to honor RFC5424. This option is ignored if +# log_config_append is set. (boolean value) +#use_syslog = false + +# Syslog facility to receive log lines. This option is ignored if +# log_config_append is set. (string value) +#syslog_log_facility = LOG_USER + +# Log output to standard error. This option is ignored if +# log_config_append is set. (boolean value) +#use_stderr = true + +# Format string to use for log messages with context. (string value) +#logging_context_format_string = %(asctime)s.%(msecs)03d %(process)d %(levelname)s %(name)s [%(request_id)s %(user_identity)s] %(instance)s%(message)s + +# Format string to use for log messages when context is undefined. +# (string value) +#logging_default_format_string = %(asctime)s.%(msecs)03d %(process)d %(levelname)s %(name)s [-] %(instance)s%(message)s + +# Additional data to append to log message when logging level for the +# message is DEBUG. (string value) +#logging_debug_format_suffix = %(funcName)s %(pathname)s:%(lineno)d + +# Prefix each line of exception output with this format. (string +# value) +#logging_exception_prefix = %(asctime)s.%(msecs)03d %(process)d ERROR %(name)s %(instance)s + +# Defines the format string for %(user_identity)s that is used in +# logging_context_format_string. (string value) +#logging_user_identity_format = %(user)s %(tenant)s %(domain)s %(user_domain)s %(project_domain)s + +# List of package logging levels in logger=LEVEL pairs. This option is +# ignored if log_config_append is set. (list value) +#default_log_levels = amqp=WARN,amqplib=WARN,boto=WARN,qpid=WARN,sqlalchemy=WARN,suds=INFO,oslo.messaging=INFO,iso8601=WARN,requests.packages.urllib3.connectionpool=WARN,urllib3.connectionpool=WARN,websocket=WARN,requests.packages.urllib3.util.retry=WARN,urllib3.util.retry=WARN,keystonemiddleware=WARN,routes.middleware=WARN,stevedore=WARN,taskflow=WARN,keystoneauth=WARN,oslo.cache=INFO,dogpile.core.dogpile=INFO + +# Enables or disables publication of error events. (boolean value) +#publish_errors = false + +# The format for an instance that is passed with the log message. +# (string value) +#instance_format = "[instance: %(uuid)s] " + +# The format for an instance UUID that is passed with the log message. +# (string value) +#instance_uuid_format = "[instance: %(uuid)s] " + +# Enables or disables fatal status of deprecations. (boolean value) +#fatal_deprecations = false diff --git a/faafo/etc/oslo-config-generator/faafo.conf b/faafo/etc/oslo-config-generator/faafo.conf new file mode 100644 index 0000000..b79fedd --- /dev/null +++ b/faafo/etc/oslo-config-generator/faafo.conf @@ -0,0 +1,6 @@ +[DEFAULT] +output_file = etc/faafo.conf.sample +namespace = faafo.worker +namespace = faafo.api +namespace = oslo.log +namespace = faafo.queues diff --git a/faafo/faafo/__init__.py b/faafo/faafo/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/faafo/faafo/api/__init__.py b/faafo/faafo/api/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/faafo/faafo/api/service.py b/faafo/faafo/api/service.py new file mode 100644 index 0000000..8a2479c --- /dev/null +++ b/faafo/faafo/api/service.py @@ -0,0 +1,146 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import base64 +import copy +import cStringIO +from pkg_resources import resource_filename + +import flask +import flask.ext.restless +import flask.ext.sqlalchemy +from flask_bootstrap import Bootstrap +from kombu import Connection +from kombu.pools import producers +from oslo_config import cfg +from oslo_log import log +from PIL import Image +from sqlalchemy.dialects import mysql + +from faafo import queues +from faafo import version + +LOG = log.getLogger('faafo.api') +CONF = cfg.CONF + +api_opts = [ + cfg.StrOpt('listen-address', + default='0.0.0.0', + help='Listen address.'), + cfg.IntOpt('bind-port', + default='80', + help='Bind port.'), + cfg.StrOpt('database-url', + default='sqlite:////tmp/sqlite.db', + help='Database connection URL.') +] + +CONF.register_opts(api_opts) + +log.register_options(CONF) +log.set_defaults() + +CONF(project='api', prog='faafo-api', + default_config_files=['/etc/faafo/faafo.conf'], + version=version.version_info.version_string()) + +log.setup(CONF, 'api', + version=version.version_info.version_string()) + +template_path = resource_filename(__name__, "templates") +app = flask.Flask('faafo.api', template_folder=template_path) +app.config['DEBUG'] = CONF.debug +app.config['SQLALCHEMY_DATABASE_URI'] = CONF.database_url +db = flask.ext.sqlalchemy.SQLAlchemy(app) +Bootstrap(app) + + +def list_opts(): + """Entry point for oslo-config-generator.""" + return [(None, copy.deepcopy(api_opts))] + + +class Fractal(db.Model): + uuid = db.Column(db.String(36), primary_key=True) + checksum = db.Column(db.String(256), unique=True) + url = db.Column(db.String(256), nullable=True) + duration = db.Column(db.Float) + size = db.Column(db.Integer, nullable=True) + width = db.Column(db.Integer, nullable=False) + height = db.Column(db.Integer, nullable=False) + iterations = db.Column(db.Integer, nullable=False) + xa = db.Column(db.Float, nullable=False) + xb = db.Column(db.Float, nullable=False) + ya = db.Column(db.Float, nullable=False) + yb = db.Column(db.Float, nullable=False) + + if CONF.database_url.startswith('mysql'): + LOG.debug('Using MySQL database backend') + image = db.Column(mysql.MEDIUMBLOB, nullable=True) + else: + image = db.Column(db.LargeBinary, nullable=True) + + generated_by = db.Column(db.String(256), nullable=True) + + def __repr__(self): + return '' % self.uuid + + +db.create_all() +manager = flask.ext.restless.APIManager(app, flask_sqlalchemy_db=db) +connection = Connection(CONF.transport_url) + + +@app.route('/', methods=['GET']) +@app.route('/index', methods=['GET']) +@app.route('/index/', methods=['GET']) +def index(page=1): + fractals = Fractal.query.filter( + (Fractal.checksum != None) & (Fractal.size != None)).paginate( # noqa + page, 5, error_out=False) + return flask.render_template('index.html', fractals=fractals) + + +@app.route('/fractal/', methods=['GET']) +def get_fractal(fractalid): + fractal = Fractal.query.filter_by(uuid=fractalid).first() + if not fractal: + response = flask.jsonify({'code': 404, + 'message': 'Fracal not found'}) + response.status_code = 404 + else: + image_data = base64.b64decode(fractal.image) + image = Image.open(cStringIO.StringIO(image_data)) + output = cStringIO.StringIO() + image.save(output, "PNG") + image.seek(0) + response = flask.make_response(output.getvalue()) + response.content_type = "image/png" + + return response + + +def generate_fractal(**kwargs): + with producers[connection].acquire(block=True) as producer: + producer.publish(kwargs['result'], + serializer='json', + exchange=queues.task_exchange, + declare=[queues.task_exchange], + routing_key='normal') + + +def main(): + manager.create_api(Fractal, methods=['GET', 'POST', 'DELETE', 'PUT'], + postprocessors={'POST': [generate_fractal]}, + exclude_columns=['image'], + url_prefix='/v1') + app.run(host=CONF.listen_address, port=CONF.bind_port) diff --git a/faafo/faafo/api/templates/index.html b/faafo/faafo/api/templates/index.html new file mode 100644 index 0000000..cc4a44a --- /dev/null +++ b/faafo/faafo/api/templates/index.html @@ -0,0 +1,60 @@ +{% extends "bootstrap/base.html" %} +{% block title %}First App Application for OpenStack{% endblock %} +{% from "bootstrap/pagination.html" import render_pagination %} + +{% block content %} +{{render_pagination(fractals)}} +{% for fractal in fractals.items %} +
+
+ + + +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
UUID{{ fractal.uuid }}
Duration{{ fractal.duration }} seconds
Dimensions{{ fractal.width }} x {{ fractal.height }} px
Iterations{{ fractal.iterations }}
Parameters +
xa = {{ fractal.xa }}
+xb = {{ fractal.xb }}
+ya = {{ fractal.ya }}
+yb = {{ fractal.yb }}
+
Filesize{{ fractal.size}} bytes
Checksum
{{ fractal.checksum }}
Generated by{{ fractal.generated_by }}
+
+
+{% endfor %} +{{render_pagination(fractals)}} +{% endblock %} diff --git a/faafo/faafo/queues.py b/faafo/faafo/queues.py new file mode 100644 index 0000000..4e5a6fd --- /dev/null +++ b/faafo/faafo/queues.py @@ -0,0 +1,32 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import copy + +import kombu +from oslo_config import cfg + +task_exchange = kombu.Exchange('tasks', type='direct') +task_queue = kombu.Queue('normal', task_exchange, routing_key='normal') + +queues_opts = [ + cfg.StrOpt('transport-url', + default='amqp://guest:guest@localhost:5672//', + help='AMQP connection URL.') +] + +cfg.CONF.register_opts(queues_opts) + + +def list_opts(): + """Entry point for oslo-config-generator.""" + return [(None, copy.deepcopy(queues_opts))] diff --git a/faafo/faafo/version.py b/faafo/faafo/version.py new file mode 100644 index 0000000..7a68690 --- /dev/null +++ b/faafo/faafo/version.py @@ -0,0 +1,15 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import pbr.version + +version_info = pbr.version.VersionInfo('faafo') diff --git a/faafo/faafo/worker/__init__.py b/faafo/faafo/worker/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/faafo/faafo/worker/service.py b/faafo/faafo/worker/service.py new file mode 100644 index 0000000..d1bb193 --- /dev/null +++ b/faafo/faafo/worker/service.py @@ -0,0 +1,156 @@ +#!/usr/bin/env python + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# based on http://code.activestate.com/recipes/577120-julia-fractals/ + +import base64 +import copy +import hashlib +import json +import os +from PIL import Image +import random +import socket +import tempfile +import time + +from kombu.mixins import ConsumerMixin +from oslo_config import cfg +from oslo_log import log +import requests + +from faafo import queues + +LOG = log.getLogger('faafo.worker') +CONF = cfg.CONF + + +worker_opts = { + cfg.StrOpt('endpoint-url', + default='http://localhost', + help='API connection URL') +} + +CONF.register_opts(worker_opts) + + +def list_opts(): + """Entry point for oslo-config-generator.""" + return [(None, copy.deepcopy(worker_opts))] + + +class JuliaSet(object): + + def __init__(self, width, height, xa=-2.0, xb=2.0, ya=-1.5, yb=1.5, + iterations=255): + self.xa = xa + self.xb = xb + self.ya = ya + self.yb = yb + self.iterations = iterations + self.width = width + self.height = height + self.draw() + + def draw(self): + self.image = Image.new("RGB", (self.width, self.height)) + c, z = self._set_point() + for y in range(self.height): + zy = y * (self.yb - self.ya) / (self.height - 1) + self.ya + for x in range(self.width): + zx = x * (self.xb - self.xa) / (self.width - 1) + self.xa + z = zx + zy * 1j + for i in range(self.iterations): + if abs(z) > 2.0: + break + z = z * z + c + self.image.putpixel((x, y), + (i % 8 * 32, i % 16 * 16, i % 32 * 8)) + + def get_file(self): + with tempfile.NamedTemporaryFile(delete=False) as fp: + self.image.save(fp, "PNG") + return fp.name + + def _set_point(self): + random.seed() + while True: + cx = random.random() * (self.xb - self.xa) + self.xa + cy = random.random() * (self.yb - self.ya) + self.ya + c = cx + cy * 1j + z = c + for i in range(self.iterations): + if abs(z) > 2.0: + break + z = z * z + c + if i > 10 and i < 100: + break + + return (c, z) + + +class Worker(ConsumerMixin): + + def __init__(self, connection): + self.connection = connection + + def get_consumers(self, Consumer, channel): + return [Consumer(queues=queues.task_queue, + accept=['json'], + callbacks=[self.process])] + + def process(self, task, message): + LOG.info("processing task %s" % task['uuid']) + LOG.debug(task) + start_time = time.time() + juliaset = JuliaSet(task['width'], + task['height'], + task['xa'], + task['xb'], + task['ya'], + task['yb'], + task['iterations']) + elapsed_time = time.time() - start_time + LOG.info("task %s processed in %f seconds" % + (task['uuid'], elapsed_time)) + + filename = juliaset.get_file() + LOG.debug("saved result of task %s to temporary file %s" % + (task['uuid'], filename)) + with open(filename, "rb") as fp: + size = os.fstat(fp.fileno()).st_size + image = base64.b64encode(fp.read()) + checksum = hashlib.sha256(open(filename, 'rb').read()).hexdigest() + os.remove(filename) + LOG.debug("removed temporary file %s" % filename) + + result = { + 'uuid': task['uuid'], + 'duration': elapsed_time, + 'image': image, + 'checksum': checksum, + 'size': size, + 'generated_by': socket.gethostname() + } + + # NOTE(berendt): only necessary when using requests < 2.4.2 + headers = {'Content-type': 'application/json', + 'Accept': 'text/plain'} + + requests.put("%s/v1/fractal/%s" % + (CONF.endpoint_url, str(task['uuid'])), + json.dumps(result), headers=headers) + + message.ack() + return result diff --git a/faafo/requirements.txt b/faafo/requirements.txt new file mode 100644 index 0000000..55d2597 --- /dev/null +++ b/faafo/requirements.txt @@ -0,0 +1,17 @@ +pbr>=1.6 +pytz +positional +iso8601 +anyjson>=0.3.3 +eventlet>=0.17.4 +PyMySQL>=0.6.2,<0.7 # 0.7 design change breaks faafo, MIT License +Pillow==2.4.0 # MIT +requests>=2.5.2 +Flask-Bootstrap +Flask>=0.10,<1.0 +flask-restless +flask-sqlalchemy +oslo.config>=2.3.0 # Apache-2.0 +oslo.log>=1.8.0 # Apache-2.0 +PrettyTable>=0.7,<0.8 +kombu>=3.0.7 diff --git a/faafo/setup.cfg b/faafo/setup.cfg new file mode 100644 index 0000000..47e621e --- /dev/null +++ b/faafo/setup.cfg @@ -0,0 +1,54 @@ +[metadata] +name = faafo +summary = First App Application for OpenStack +description-file = + README.rst +author = OpenStack Documentation +author-email = openstack-doc@lists.openstack.org +home-page = http://docs.openstack.org/developer/faafo/ +classifier = + Environment :: OpenStack + Intended Audience :: Information Technology + Intended Audience :: System Administrators + License :: OSI Approved :: Apache Software License + Operating System :: POSIX :: Linux + Programming Language :: Python + Programming Language :: Python :: 2 + Programming Language :: Python :: 2.7 + +[files] +packages = + faafo +scripts = + bin/faafo + bin/faafo-worker +extra_files = + faafo/api/templates/index.html +data_files = + /etc/faafo = etc/faafo.conf + +[global] +setup-hooks = + pbr.hooks.setup_hook + +[entry_points] +console_scripts = + faafo-api = faafo.api.service:main +oslo.config.opts = + faafo.api = faafo.api.service:list_opts + faafo.worker = faafo.worker.service:list_opts + faafo.queues= faafo.queues:list_opts + +[build_sphinx] +source-dir = doc/source +build-dir = doc/build +all_files = 1 + +[upload_sphinx] +upload-dir = doc/build/html + +[wheel] +universal = 1 + +[pbr] +warnerrors = true diff --git a/faafo/setup.py b/faafo/setup.py new file mode 100644 index 0000000..ee06f22 --- /dev/null +++ b/faafo/setup.py @@ -0,0 +1,29 @@ +#!/usr/bin/env python +# Copyright (c) 2013 Hewlett-Packard Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import setuptools + +# In python < 2.7.4, a lazy loading of package `pbr` will break +# setuptools if some other modules registered functions in `atexit`. +# solution from: http://bugs.python.org/issue15881#msg170215 +try: + import multiprocessing # noqa +except ImportError: + pass + +setuptools.setup( + setup_requires=['pbr'], + pbr=True) diff --git a/faafo/test-requirements.txt b/faafo/test-requirements.txt new file mode 100644 index 0000000..3592358 --- /dev/null +++ b/faafo/test-requirements.txt @@ -0,0 +1,2 @@ +hacking<0.11,>=0.10 +sphinx>=1.1.2,!=1.2.0,!=1.3b1,<1.3 diff --git a/faafo/tox.ini b/faafo/tox.ini new file mode 100644 index 0000000..5f9d4d0 --- /dev/null +++ b/faafo/tox.ini @@ -0,0 +1,27 @@ +[tox] +minversion = 1.6 +envlist = pep8 +skipsdist = True + +[testenv] +usedevelop = True +deps = -r{toxinidir}/test-requirements.txt + -r{toxinidir}/requirements.txt +install_command = pip install -U {opts} {packages} + +[testenv:venv] +commands = {posargs} + +[testenv:docs] +commands = python setup.py build_sphinx + +[testenv:genconfig] +commands = + oslo-config-generator --config-file etc/oslo-config-generator/faafo.conf + +[testenv:pep8] +commands = flake8 {posargs} + +[flake8] +show-source = True +exclude=.venv,.git,.tox,*egg*,build,*openstack/common*