Mini Shell

# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import concurrent
import time

import testtools
from testscenarios import load_tests_apply_scenarios as load_tests  # noqa

import openstack
import openstack.cloud
from openstack.cloud import meta
from openstack import exceptions
from openstack.tests import fakes
from openstack.tests.unit import base
from openstack.tests.unit.cloud import test_port


# Mock out the gettext function so that the task schema can be copypasta
def _(msg):
    return msg


_TASK_PROPERTIES = {
    "id": {
        "description": _("An identifier for the task"),
        "pattern": _('^([0-9a-fA-F]){8}-([0-9a-fA-F]){4}-([0-9a-fA-F]){4}'
                     '-([0-9a-fA-F]){4}-([0-9a-fA-F]){12}$'),
        "type": "string"
    },
    "type": {
        "description": _("The type of task represented by this content"),
        "enum": [
            "import",
        ],
        "type": "string"
    },
    "status": {
        "description": _("The current status of this task"),
        "enum": [
            "pending",
            "processing",
            "success",
            "failure"
        ],
        "type": "string"
    },
    "input": {
        "description": _("The parameters required by task, JSON blob"),
        "type": ["null", "object"],
    },
    "result": {
        "description": _("The result of current task, JSON blob"),
        "type": ["null", "object"],
    },
    "owner": {
        "description": _("An identifier for the owner of this task"),
        "type": "string"
    },
    "message": {
        "description": _("Human-readable informative message only included"
                         " when appropriate (usually on failure)"),
        "type": "string",
    },
    "expires_at": {
        "description": _("Datetime when this resource would be"
                         " subject to removal"),
        "type": ["null", "string"]
    },
    "created_at": {
        "description": _("Datetime when this resource was created"),
        "type": "string"
    },
    "updated_at": {
        "description": _("Datetime when this resource was updated"),
        "type": "string"
    },
    'self': {'type': 'string'},
    'schema': {'type': 'string'}
}
_TASK_SCHEMA = dict(
    name='Task', properties=_TASK_PROPERTIES,
    additionalProperties=False,
)


class TestMemoryCache(base.TestCase):

    def setUp(self):
        super(TestMemoryCache, self).setUp(
            cloud_config_fixture='clouds_cache.yaml')

    def _image_dict(self, fake_image):
        return self.cloud._normalize_image(meta.obj_to_munch(fake_image))

    def _munch_images(self, fake_image):
        return self.cloud._normalize_images([fake_image])

    def test_openstack_cloud(self):
        self.assertIsInstance(self.cloud, openstack.connection.Connection)

    def test_list_projects_v3(self):
        project_one = self._get_project_data()
        project_two = self._get_project_data()
        project_list = [project_one, project_two]

        first_response = {'projects': [project_one.json_response['project']]}
        second_response = {'projects': [p.json_response['project']
                                        for p in project_list]}

        mock_uri = self.get_mock_url(
            service_type='identity', resource='projects',
            base_url_append='v3')

        self.register_uris([
            dict(method='GET', uri=mock_uri, status_code=200,
                 json=first_response),
            dict(method='GET', uri=mock_uri, status_code=200,
                 json=second_response)])

        self.assertEqual(
            self.cloud._normalize_projects(
                meta.obj_list_to_munch(first_response['projects'])),
            self.cloud.list_projects())
        self.assertEqual(
            self.cloud._normalize_projects(
                meta.obj_list_to_munch(first_response['projects'])),
            self.cloud.list_projects())
        # invalidate the list_projects cache
        self.cloud.list_projects.invalidate(self.cloud)
        # ensure the new values are now retrieved
        self.assertEqual(
            self.cloud._normalize_projects(
                meta.obj_list_to_munch(second_response['projects'])),
            self.cloud.list_projects())
        self.assert_calls()

    def test_list_projects_v2(self):
        self.use_keystone_v2()
        project_one = self._get_project_data(v3=False)
        project_two = self._get_project_data(v3=False)
        project_list = [project_one, project_two]

        first_response = {'tenants': [project_one.json_response['tenant']]}
        second_response = {'tenants': [p.json_response['tenant']
                                       for p in project_list]}

        mock_uri = self.get_mock_url(
            service_type='identity', interface='admin', resource='tenants')

        self.register_uris([
            dict(method='GET', uri=mock_uri, status_code=200,
                 json=first_response),
            dict(method='GET', uri=mock_uri, status_code=200,
                 json=second_response)])

        self.assertEqual(
            self.cloud._normalize_projects(
                meta.obj_list_to_munch(first_response['tenants'])),
            self.cloud.list_projects())
        self.assertEqual(
            self.cloud._normalize_projects(
                meta.obj_list_to_munch(first_response['tenants'])),
            self.cloud.list_projects())
        # invalidate the list_projects cache
        self.cloud.list_projects.invalidate(self.cloud)
        # ensure the new values are now retrieved
        self.assertEqual(
            self.cloud._normalize_projects(
                meta.obj_list_to_munch(second_response['tenants'])),
            self.cloud.list_projects())
        self.assert_calls()

    def test_list_servers_no_herd(self):
        self.cloud._SERVER_AGE = 2
        fake_server = fakes.make_fake_server('1234', 'name')
        self.register_uris([
            self.get_nova_discovery_mock_dict(),
            dict(method='GET',
                 uri=self.get_mock_url(
                     'compute', 'public', append=['servers', 'detail']),
                 json={'servers': [fake_server]}),
        ])
        with concurrent.futures.ThreadPoolExecutor(16) as pool:
            for i in range(16):
                pool.submit(lambda: self.cloud.list_servers(bare=True))
                # It's possible to race-condition 16 threads all in the
                # single initial lock without a tiny sleep
                time.sleep(0.001)

        self.assert_calls()

    def test_list_volumes(self):
        fake_volume = fakes.FakeVolume('volume1', 'available',
                                       'Volume 1 Display Name')
        fake_volume_dict = meta.obj_to_munch(fake_volume)
        fake_volume2 = fakes.FakeVolume('volume2', 'available',
                                        'Volume 2 Display Name')
        fake_volume2_dict = meta.obj_to_munch(fake_volume2)
        self.register_uris([
            self.get_cinder_discovery_mock_dict(),
            dict(method='GET',
                 uri=self.get_mock_url(
                     'volumev2', 'public', append=['volumes', 'detail']),
                 json={'volumes': [fake_volume_dict]}),
            dict(method='GET',
                 uri=self.get_mock_url(
                     'volumev2', 'public', append=['volumes', 'detail']),
                 json={'volumes': [fake_volume_dict, fake_volume2_dict]})])
        self.assertEqual(
            [self.cloud._normalize_volume(fake_volume_dict)],
            self.cloud.list_volumes())
        # this call should hit the cache
        self.assertEqual(
            [self.cloud._normalize_volume(fake_volume_dict)],
            self.cloud.list_volumes())
        self.cloud.list_volumes.invalidate(self.cloud)
        self.assertEqual(
            [self.cloud._normalize_volume(fake_volume_dict),
             self.cloud._normalize_volume(fake_volume2_dict)],
            self.cloud.list_volumes())
        self.assert_calls()

    def test_list_volumes_creating_invalidates(self):
        fake_volume = fakes.FakeVolume('volume1', 'creating',
                                       'Volume 1 Display Name')
        fake_volume_dict = meta.obj_to_munch(fake_volume)
        fake_volume2 = fakes.FakeVolume('volume2', 'available',
                                        'Volume 2 Display Name')
        fake_volume2_dict = meta.obj_to_munch(fake_volume2)
        self.register_uris([
            self.get_cinder_discovery_mock_dict(),
            dict(method='GET',
                 uri=self.get_mock_url(
                     'volumev2', 'public', append=['volumes', 'detail']),
                 json={'volumes': [fake_volume_dict]}),
            dict(method='GET',
                 uri=self.get_mock_url(
                     'volumev2', 'public', append=['volumes', 'detail']),
                 json={'volumes': [fake_volume_dict, fake_volume2_dict]})])
        self.assertEqual(
            [self.cloud._normalize_volume(fake_volume_dict)],
            self.cloud.list_volumes())
        self.assertEqual(
            [self.cloud._normalize_volume(fake_volume_dict),
             self.cloud._normalize_volume(fake_volume2_dict)],
            self.cloud.list_volumes())
        self.assert_calls()

    def test_create_volume_invalidates(self):
        fake_volb4 = meta.obj_to_munch(
            fakes.FakeVolume('volume1', 'available', ''))
        _id = '12345'
        fake_vol_creating = meta.obj_to_munch(
            fakes.FakeVolume(_id, 'creating', ''))
        fake_vol_avail = meta.obj_to_munch(
            fakes.FakeVolume(_id, 'available', ''))

        def now_deleting(request, context):
            fake_vol_avail['status'] = 'deleting'

        self.register_uris([
            self.get_cinder_discovery_mock_dict(),
            dict(method='GET',
                 uri=self.get_mock_url(
                     'volumev2', 'public', append=['volumes', 'detail']),
                 json={'volumes': [fake_volb4]}),
            dict(method='POST',
                 uri=self.get_mock_url(
                     'volumev2', 'public', append=['volumes']),
                 json={'volume': fake_vol_creating}),
            dict(method='GET',
                 uri=self.get_mock_url(
                     'volumev2', 'public', append=['volumes', 'detail']),
                 json={'volumes': [fake_volb4, fake_vol_creating]}),
            dict(method='GET',
                 uri=self.get_mock_url(
                     'volumev2', 'public', append=['volumes', 'detail']),
                 json={'volumes': [fake_volb4, fake_vol_avail]}),
            dict(method='GET',
                 uri=self.get_mock_url(
                     'volumev2', 'public', append=['volumes', 'detail']),
                 json={'volumes': [fake_volb4, fake_vol_avail]}),
            dict(method='DELETE',
                 uri=self.get_mock_url(
                     'volumev2', 'public', append=['volumes', _id]),
                 json=now_deleting),
            dict(method='GET',
                 uri=self.get_mock_url(
                     'volumev2', 'public', append=['volumes', 'detail']),
                 json={'volumes': [fake_volb4]})])

        self.assertEqual(
            [self.cloud._normalize_volume(fake_volb4)],
            self.cloud.list_volumes())
        volume = dict(display_name='junk_vol',
                      size=1,
                      display_description='test junk volume')
        self.cloud.create_volume(wait=True, timeout=None, **volume)
        # If cache was not invalidated, we would not see our own volume here
        # because the first volume was available and thus would already be
        # cached.
        self.assertEqual(
            [self.cloud._normalize_volume(fake_volb4),
             self.cloud._normalize_volume(fake_vol_avail)],
            self.cloud.list_volumes())
        self.cloud.delete_volume(_id)
        # And now delete and check same thing since list is cached as all
        # available
        self.assertEqual(
            [self.cloud._normalize_volume(fake_volb4)],
            self.cloud.list_volumes())
        self.assert_calls()

    def test_list_users(self):
        user_data = self._get_user_data(email='test@example.com')
        self.register_uris([
            dict(method='GET',
                 uri=self.get_mock_url(
                     service_type='identity',
                     resource='users',
                     base_url_append='v3'),
                 status_code=200,
                 json={'users': [user_data.json_response['user']]})])
        users = self.cloud.list_users()
        self.assertEqual(1, len(users))
        self.assertEqual(user_data.user_id, users[0]['id'])
        self.assertEqual(user_data.name, users[0]['name'])
        self.assertEqual(user_data.email, users[0]['email'])
        self.assert_calls()

    def test_modify_user_invalidates_cache(self):
        self.use_keystone_v2()

        user_data = self._get_user_data(email='test@example.com')
        new_resp = {'user': user_data.json_response['user'].copy()}
        new_resp['user']['email'] = 'Nope@Nope.Nope'
        new_req = {'user': {'email': new_resp['user']['email']}}

        mock_users_url = self.get_mock_url(
            service_type='identity',
            interface='admin',
            resource='users')
        mock_user_resource_url = self.get_mock_url(
            service_type='identity',
            interface='admin',
            resource='users',
            append=[user_data.user_id])

        empty_user_list_resp = {'users': []}
        users_list_resp = {'users': [user_data.json_response['user']]}
        updated_users_list_resp = {'users': [new_resp['user']]}

        # Password is None in the original create below
        user_data.json_request['user']['password'] = None

        uris_to_mock = [
            # Inital User List is Empty
            dict(method='GET', uri=mock_users_url, status_code=200,
                 json=empty_user_list_resp),
            # POST to create the user
            # GET to get the user data after POST
            dict(method='POST', uri=mock_users_url, status_code=200,
                 json=user_data.json_response,
                 validate=dict(json=user_data.json_request)),
            # List Users Call
            dict(method='GET', uri=mock_users_url, status_code=200,
                 json=users_list_resp),
            # List users to get ID for update
            # Get user using user_id from list
            # Update user
            # Get updated user
            dict(method='GET', uri=mock_users_url, status_code=200,
                 json=users_list_resp),
            dict(method='PUT', uri=mock_user_resource_url, status_code=200,
                 json=new_resp, validate=dict(json=new_req)),
            # List Users Call
            dict(method='GET', uri=mock_users_url, status_code=200,
                 json=updated_users_list_resp),
            # List User to get ID for delete
            # Get user using user_id from list
            # delete user
            dict(method='GET', uri=mock_users_url, status_code=200,
                 json=updated_users_list_resp),
            dict(method='GET', uri=mock_user_resource_url, status_code=200,
                 json=new_resp),
            dict(method='DELETE', uri=mock_user_resource_url, status_code=204),
            # List Users Call (empty post delete)
            dict(method='GET', uri=mock_users_url, status_code=200,
                 json=empty_user_list_resp)
        ]

        self.register_uris(uris_to_mock)

        # first cache an empty list
        self.assertEqual([], self.cloud.list_users())

        # now add one
        created = self.cloud.create_user(name=user_data.name,
                                         email=user_data.email)
        self.assertEqual(user_data.user_id, created['id'])
        self.assertEqual(user_data.name, created['name'])
        self.assertEqual(user_data.email, created['email'])
        # Cache should have been invalidated
        users = self.cloud.list_users()
        self.assertEqual(user_data.user_id, users[0]['id'])
        self.assertEqual(user_data.name, users[0]['name'])
        self.assertEqual(user_data.email, users[0]['email'])

        # Update and check to see if it is updated
        updated = self.cloud.update_user(user_data.user_id,
                                         email=new_resp['user']['email'])
        self.assertEqual(user_data.user_id, updated.id)
        self.assertEqual(user_data.name, updated.name)
        self.assertEqual(new_resp['user']['email'], updated.email)
        users = self.cloud.list_users()
        self.assertEqual(1, len(users))
        self.assertEqual(user_data.user_id, users[0]['id'])
        self.assertEqual(user_data.name, users[0]['name'])
        self.assertEqual(new_resp['user']['email'], users[0]['email'])
        # Now delete and ensure it disappears
        self.cloud.delete_user(user_data.user_id)
        self.assertEqual([], self.cloud.list_users())
        self.assert_calls()

    def test_list_flavors(self):
        mock_uri = '{endpoint}/flavors/detail?is_public=None'.format(
            endpoint=fakes.COMPUTE_ENDPOINT)

        uris_to_mock = [
            dict(method='GET', uri=mock_uri, json={'flavors': []}),
            dict(method='GET', uri=mock_uri,
                 json={'flavors': fakes.FAKE_FLAVOR_LIST})
        ]

        self.register_uris(uris_to_mock)

        self.assertEqual([], self.cloud.list_flavors())

        self.assertEqual([], self.cloud.list_flavors())

        fake_flavor_dicts = self.cloud._normalize_flavors(
            fakes.FAKE_FLAVOR_LIST)
        self.cloud.list_flavors.invalidate(self.cloud)
        self.assertEqual(fake_flavor_dicts, self.cloud.list_flavors())

        self.assert_calls()

    def test_list_images(self):

        self.use_glance()
        fake_image = fakes.make_fake_image(image_id='42')

        self.register_uris([
            dict(method='GET',
                 uri=self.get_mock_url('image', 'public',
                                       append=['v2', 'images']),
                 json={'images': []}),
            dict(method='GET',
                 uri=self.get_mock_url('image', 'public',
                                       append=['v2', 'images']),
                 json={'images': [fake_image]}),
        ])

        self.assertEqual([], self.cloud.list_images())
        self.assertEqual([], self.cloud.list_images())
        self.cloud.list_images.invalidate(self.cloud)
        self.assertEqual(
            self._munch_images(fake_image), self.cloud.list_images())

        self.assert_calls()

    def test_list_images_caches_deleted_status(self):
        self.use_glance()

        deleted_image_id = self.getUniqueString()
        deleted_image = fakes.make_fake_image(
            image_id=deleted_image_id, status='deleted')
        active_image_id = self.getUniqueString()
        active_image = fakes.make_fake_image(image_id=active_image_id)
        list_return = {'images': [active_image, deleted_image]}
        self.register_uris([
            dict(method='GET',
                 uri='https://image.example.com/v2/images',
                 json=list_return),
        ])

        self.assertEqual(
            [self.cloud._normalize_image(active_image)],
            self.cloud.list_images())

        self.assertEqual(
            [self.cloud._normalize_image(active_image)],
            self.cloud.list_images())

        # We should only have one call
        self.assert_calls()

    def test_cache_no_cloud_name(self):
        self.use_glance()

        self.cloud.name = None
        fi = fakes.make_fake_image(image_id=self.getUniqueString())
        fi2 = fakes.make_fake_image(image_id=self.getUniqueString())

        self.register_uris([
            dict(method='GET',
                 uri='https://image.example.com/v2/images',
                 json={'images': [fi]}),
            dict(method='GET',
                 uri='https://image.example.com/v2/images',
                 json={'images': [fi, fi2]}),
        ])

        self.assertEqual(
            self._munch_images(fi),
            self.cloud.list_images())

        # Now test that the list was cached
        self.assertEqual(
            self._munch_images(fi),
            self.cloud.list_images())

        # Invalidation too
        self.cloud.list_images.invalidate(self.cloud)
        self.assertEqual(
            [
                self.cloud._normalize_image(fi),
                self.cloud._normalize_image(fi2)
            ],
            self.cloud.list_images())

    def test_list_ports_filtered(self):
        down_port = test_port.TestPort.mock_neutron_port_create_rep['port']
        active_port = down_port.copy()
        active_port['status'] = 'ACTIVE'
        # We're testing to make sure a query string isn't passed when we're
        # caching, but that the results are still filtered.
        self.register_uris([
            dict(method='GET',
                 uri=self.get_mock_url(
                     'network', 'public', append=['v2.0', 'ports.json']),
                 json={'ports': [
                     down_port,
                     active_port,
                 ]}),
        ])
        ports = self.cloud.list_ports(filters={'status': 'DOWN'})
        self.assertCountEqual([down_port], ports)
        self.assert_calls()


class TestCacheIgnoresQueuedStatus(base.TestCase):

    scenarios = [
        ('queued', dict(status='queued')),
        ('saving', dict(status='saving')),
        ('pending_delete', dict(status='pending_delete')),
    ]

    def setUp(self):
        super(TestCacheIgnoresQueuedStatus, self).setUp(
            cloud_config_fixture='clouds_cache.yaml')
        self.use_glance()
        active_image_id = self.getUniqueString()
        self.active_image = fakes.make_fake_image(
            image_id=active_image_id, status=self.status)
        self.active_list_return = {'images': [self.active_image]}
        steady_image_id = self.getUniqueString()
        self.steady_image = fakes.make_fake_image(image_id=steady_image_id)
        self.steady_list_return = {
            'images': [self.active_image, self.steady_image]}

    def test_list_images_ignores_pending_status(self):

        self.register_uris([
            dict(method='GET',
                 uri='https://image.example.com/v2/images',
                 json=self.active_list_return),
            dict(method='GET',
                 uri='https://image.example.com/v2/images',
                 json=self.steady_list_return),
        ])

        self.assertEqual(
            [self.cloud._normalize_image(self.active_image)],
            self.cloud.list_images())

        # Should expect steady_image to appear if active wasn't cached
        self.assertEqual(
            [
                self.cloud._normalize_image(self.active_image),
                self.cloud._normalize_image(self.steady_image)
            ],
            self.cloud.list_images())


class TestCacheSteadyStatus(base.TestCase):

    scenarios = [
        ('active', dict(status='active')),
        ('killed', dict(status='killed')),
    ]

    def setUp(self):
        super(TestCacheSteadyStatus, self).setUp(
            cloud_config_fixture='clouds_cache.yaml')
        self.use_glance()
        active_image_id = self.getUniqueString()
        self.active_image = fakes.make_fake_image(
            image_id=active_image_id, status=self.status)
        self.active_list_return = {'images': [self.active_image]}

    def test_list_images_caches_steady_status(self):

        self.register_uris([
            dict(method='GET',
                 uri='https://image.example.com/v2/images',
                 json=self.active_list_return),
        ])

        self.assertEqual(
            [self.cloud._normalize_image(self.active_image)],
            self.cloud.list_images())

        self.assertEqual(
            [self.cloud._normalize_image(self.active_image)],
            self.cloud.list_images())

        # We should only have one call
        self.assert_calls()


class TestBogusAuth(base.TestCase):

    def setUp(self):
        super(TestBogusAuth, self).setUp(
            cloud_config_fixture='clouds_cache.yaml')

    def test_get_auth_bogus(self):
        with testtools.ExpectedException(exceptions.ConfigException):
            openstack.connect(
                cloud='_bogus_test_', config=self.config)