marvin_refactor: new package structure for marvin

legacy: all old integration libraries go here
generate: all the code used to autogenerate marvin libraries
docs: documentation
data: data objects representing what were earlier service dictionaries

tools/marvin/marvin/data/__init__.py

@@ -0,0 +1,16 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.

tools/marvin/marvin/data/account.py

@@ -0,0 +1,38 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import factory
+from marvin.factory.AccountFactory import AccountFactory
+from marvin.utils import random_gen
+
+class UserAccountFactory(AccountFactory):
+
+    accounttype = 0
+    firstname = factory.Sequence(lambda n: random_gen())
+    lastname = factory.Sequence(lambda n: random_gen())
+    email = factory.LazyAttribute(lambda e: '{0}.{1}@cloudstack.org'.format(e.firstname, e.lastname).lower())
+    username = factory.Sequence(lambda n: random_gen())
+    password = 'password'
+
+
+class AdminAccountFactory(UserAccountFactory):
+    accounttype = 1
+
+
+class DomainAdminFactory(UserAccountFactory):
+    accounttype = 2
+    domainid = None

tools/marvin/marvin/data/cluster.py

@@ -0,0 +1,30 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import factory
+from marvin.utils import random_gen
+from marvin.factory.ClusterFactory import ClusterFactory
+
+class XenClusterFactory(ClusterFactory):
+    clustername = factory.Sequence(lambda n: "xencluster" + random_gen())
+    clustertype = "XenServer"
+    hypervisor = "XenServer"
+
+class KvmClusterFactory(ClusterFactory):
+    clustername = factory.Sequence(lambda n: "kvmcluster" + random_gen())
+    clustertype = "KVM"
+    hypervisor = "KVM"

tools/marvin/marvin/data/diskoffering.py

@@ -0,0 +1,34 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import factory
+from marvin.factory.DiskOfferingFactory import DiskOfferingFactory
+from marvin.utils import random_gen
+
+class SharedDiskOfferingFactory(DiskOfferingFactory):
+
+    displaytext = "SharedDiskOffering"
+    name = factory.Sequence(lambda n : "SharedDiskOffering" + random_gen())
+    storagetype = "shared"
+    disksize = 10 #MB
+
+class LocalDiskOfferingFactory(DiskOfferingFactory):
+
+    displaytext = "LocalDiskOffering"
+    name = factory.Sequence(lambda n : "LocalDiskOffering" + random_gen())
+    storagetype = "local"
+    disksize = 10 #MB

tools/marvin/marvin/data/firewallrule.py

@@ -0,0 +1,30 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from marvin.factory.FirewallRuleFactory import FirewallRuleFactory
+
+class SshFirewallRuleFactory(FirewallRuleFactory):
+    protocol = 'tcp'
+    startport = 22
+    endport = 22
+    cidrlist = '0.0.0.0/0'
+
+class HttpFirewallRuleFactory(FirewallRuleFactory):
+    protocol = 'tcp'
+    startport = 80
+    endport = 80
+    cidrlist = '0.0.0.0/0'

tools/marvin/marvin/data/host.py

@@ -0,0 +1,31 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from marvin.factory.HostFactory import HostFactory
+
+class XenserverHostFactory(HostFactory):
+
+    hypervisor = 'XenServer'
+    password = 'password'
+    username = 'root'
+
+
+class KvmHostFactory(HostFactory):
+
+    hypervisor = 'KVM'
+    password = 'password'
+    username = 'root'

tools/marvin/marvin/data/networkoffering.py

@@ -0,0 +1,73 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import factory
+from marvin.factory.NetworkOfferingFactory import NetworkOfferingFactory
+from marvin.utils import random_gen
+
+
+class DefaultIsolatedNetworkOfferingWithSourceNatServiceFactory(NetworkOfferingFactory):
+    #FIXME: Service Capability Lists with CapabilityTypes (ElasticIP, RvR etc) needs handling
+
+    displaytext = factory.Sequence(lambda n : "DefaultIsolatedNetworkOfferingWithSourceNatService" + random_gen())
+    name = factory.Sequence(lambda n : "DefaultIsolatedNetworkOfferingWithSourceNatService" + random_gen())
+    supportedservices = "Lb,Dns,PortForwarding,StaticNat,Dhcp,Firewall,Vpn,UserData,SourceNat"
+    traffictype = "GUEST"
+    availability = "Optional"
+    guestiptype = "Isolated"
+
+    specifyVlan = False
+    specifyIpRanges = False
+    isPersistent = False
+    conserveMode = True
+
+    serviceProviderList = []
+    for service in map(lambda l: l.strip(' '), supportedservices.split(',')):
+        serviceProviderList.append(
+            {
+                'service': service,
+                'provider': 'VirtualRouter'
+            }
+        )
+
+
+class DefaultSharedNetworkOfferingWithSGServiceFactory(NetworkOfferingFactory):
+
+    displaytext = factory.Sequence(lambda n : "DefaultSharedNetworkOfferingWithSGService" + random_gen())
+    name = factory.Sequence(lambda n : "DefaultSharedNetworkOfferingWithSGService" + random_gen())
+    availability = "Optional"
+    supportedservices = "SecurityGroup,Dns,Dhcp,UserData"
+    guestiptype = "Shared"
+    traffictype = "GUEST"
+
+    specifyVlan = True
+    specifyIpRanges = True
+    isPersistent = False
+    conserveMode = True
+
+    serviceProviderList = []
+    for service in map(lambda l: l.strip(' '), supportedservices.split(',')):
+        if service == 'SecurityGroup':
+            provider = 'SecurityGroupProvider'
+        else:
+            provider = 'VirtualRouter'
+        serviceProviderList.append(
+            {
+                'service': service,
+                'provider': provider
+            }
+        )

tools/marvin/marvin/data/serviceoffering.py

@@ -0,0 +1,27 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import factory
+from marvin.factory.ServiceOfferingFactory import ServiceOfferingFactory
+from marvin.utils import random_gen
+
+class SmallServiceOfferingFactory(ServiceOfferingFactory):
+    cpunumber = 1
+    cpuspeed = 100 #Mhz
+    memory = 100 #MB
+    displaytext = "Small Service Offering"
+    name = factory.Sequence(lambda n: "SmallServiceOffering" + random_gen())

tools/marvin/marvin/data/template.py

@@ -0,0 +1,25 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from marvin.factory.TemplateFactory import TemplateFactory
+
+class DefaultBuiltInTemplateFactory(TemplateFactory):
+    ostype = 'CentOS 5.3 (64-bit)'
+    displaytext = 'CentOS 5.3 (64-bit)'
+    name = 'CentOS 5.3 (64-bit)'
+    format = 'VHD'
+    hypervisor = 'XenServer'

tools/marvin/marvin/data/user.py

@@ -0,0 +1,50 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import factory
+from marvin.factory.UserFactory import UserFactory
+from marvin.data.account import UserAccountFactory
+from marvin.utils import random_gen
+
+class UserFactory(UserFactory):
+
+    firstname = factory.Sequence(lambda n: random_gen())
+    lastname = factory.Sequence(lambda n: random_gen())
+    email = factory.LazyAttribute(lambda e: '{0}.{1}@cloudstack.org'.format(e.firstname, e.lastname).lower())
+    username = factory.Sequence(lambda n: random_gen())
+    password = 'password'
+    account = factory.SubFactory(UserAccountFactory,
+        apiclient=factory.SelfAttribute('..apiclient'),
+        accounttype=0,
+        firstname=factory.SelfAttribute('..firstname'),
+        lastname=factory.SelfAttribute('..lastname'),
+        email=factory.SelfAttribute('..email'),
+        password=factory.SelfAttribute('..password'),
+        username=factory.SelfAttribute('..username'),
+    )
+
+class AdminUserFactory(UserFactory):
+
+    account = factory.SubFactory(UserAccountFactory,
+        apiclient=factory.SelfAttribute('..apiclient'),
+        accounttype=1,
+        firstname=factory.SelfAttribute('..firstname'),
+        lastname=factory.SelfAttribute('..lastname'),
+        email=factory.SelfAttribute('..email'),
+        password=factory.SelfAttribute('..password'),
+        username=factory.SelfAttribute('..username'),
+    )

tools/marvin/marvin/data/vm.py

@@ -0,0 +1,34 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import factory
+from marvin.factory.VirtualMachineFactory import VirtualMachineFactory
+from marvin.utils import random_gen
+
+
+class VirtualMachineIsolatedNetwork(VirtualMachineFactory):
+    """
+    Create a virtualmachine in an isolated network typically in an advanced zone
+
+    Uses a serviceoffering of tinyInstance
+    Uses a builtin template available
+    Deploys in the first zone available
+    """
+
+    serviceofferingid = None
+    templateid = None
+    zoneid = None

tools/marvin/marvin/data/zone.py

@@ -0,0 +1,33 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import factory
+from marvin.factory.ZoneFactory import ZoneFactory
+from marvin.utils import random_gen
+
+class AdvancedZoneFactory(ZoneFactory):
+    name = factory.Sequence(lambda n: "advzone" + random_gen())
+    networktype = "Advanced"
+    dns1 = "8.8.8.8"
+    internaldns1 = "8.8.8.8"
+
+
+class BasicZoneFactory(ZoneFactory):
+    name = factory.Sequence(lambda n: "basiczone" + random_gen())
+    networktype = "Basic"
+    dns1 = "8.8.8.8"
+    internaldns1 = "8.8.8.8"

tools/marvin/marvin/docs/__init__.py

@@ -0,0 +1,16 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.

tools/marvin/marvin/docs/errata.md

@@ -0,0 +1,22 @@
+## Idea Stack
+
+### Bugs
+
+- **marvin.sync and xml compilation produce different versions of cloudstackAPI**
+- marvin build now requires inflect which will cause -Pdeveloper profile to break for the first time
+- Entities should include @docstring for optional arguments in their actions() methods. **kwargs is confusing
+- Dissociate the grammar list to make it extensible via a properties file
+- Handle APIs that need parameters but dont have a required args list because multiple sets of args form a required list
+	- eg: disableAccount (either provide id (account) or accoutname and domainid)
+- XML precache required for factory and base generation [CLOUDSTACK-4589](https://issues.apache.org/jira//browse/CLOUDSTACK-4589)
+- Remove marvin dependency with apidoc build. Provide precache json [CLOUDSTACK-4589](https://issues.apache.org/jira//browse/CLOUDSTACK-4589)
+- Better sync functionality
+- Bump up version to 0.2.0
+- Improved cleanup support
+
+### Features
+- Export deployment to JSON [CLOUDSTACK-4590](https://issues.apache.org/jira//browse/CLOUDSTACK-4590)
+- nose2 and unittest2 support [CLOUDSTACK-4591](https://issues.apache.org/jira//browse/CLOUDSTACK-4591)
+- Use distutils
+- Python pip repository for cloudstack-marvin
+- DSL for marvin using Behave [CLOUDSTACK-1952](https://issues.apache.org/jira/browse/CLOUDSTACK-1952)

tools/marvin/marvin/generate/__init__.py

@@ -0,0 +1,21 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+
+# The generate package includes various transformations applied to
+# the CloudStack DSL to convert it to the working model of marvin's
+# test libraries

tools/marvin/marvin/legacy/__init__.py

@@ -0,0 +1,19 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# legacy marvin libraries that will soon be deprecated in
+# future versions