etan39
/
cloudstack
mirror of https://github.com/apache/cloudstack.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

marvin+apidiscovery: Extend API discovery plugin 

API discovery plugin will return embedded entities for marvin to
discovery and generate it's API classes.

Signed-off-by: Prasanna Santhanam <tsp@apache.org>
This commit is contained in:
Prasanna Santhanam 2013-03-26 18:29:23 +05:30
parent d4dc264917
commit 5d67c98e5b
10 changed files with 295 additions and 110 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
client/pom.xml
View File

@ -90,11 +90,6 @@
<artifactId>cloud-plugin-hypervisor-baremetal</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.apache.cloudstack</groupId>
<artifactId>cloud-plugin-hypervisor-ucs</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.apache.cloudstack</groupId>
<artifactId>cloud-plugin-hypervisor-ovm</artifactId>

4
plugins/api/discovery/src/org/apache/cloudstack/api/response/ApiDiscoveryResponse.java
View File

@ -47,6 +47,9 @@ public class ApiDiscoveryResponse extends BaseResponse {
@SerializedName(ApiConstants.RESPONSE) @Param(description="api response fields", responseObject = ApiResponseResponse.class)
private Set<ApiResponseResponse> apiResponse;
@SerializedName(ApiConstants.TYPE) @Param(description="response field type")
private String type;
public ApiDiscoveryResponse(){
params = new HashSet<ApiParameterResponse>();
apiResponse = new HashSet<ApiResponseResponse>();
@ -81,6 +84,7 @@ public class ApiDiscoveryResponse extends BaseResponse {
this.isAsync = isAsync;
}
public boolean getAsync() {
return isAsync;
}

15
plugins/api/discovery/src/org/apache/cloudstack/api/response/ApiResponseResponse.java
View File

@ -16,11 +16,14 @@
// under the License.
package org.apache.cloudstack.api.response;
import org.apache.cloudstack.api.ApiConstants;
import com.cloud.serializer.Param;
import com.google.gson.annotations.SerializedName;
import org.apache.cloudstack.api.ApiConstants;
import org.apache.cloudstack.api.BaseResponse;
import java.util.HashSet;
import java.util.Set;
public class ApiResponseResponse extends BaseResponse {
@SerializedName(ApiConstants.NAME) @Param(description="the name of the api response field")
private String name;
@ -31,6 +34,9 @@ public class ApiResponseResponse extends BaseResponse {
@SerializedName(ApiConstants.TYPE) @Param(description="response field type")
private String type;
@SerializedName(ApiConstants.RESPONSE) @Param(description="api response fields")
private Set<ApiResponseResponse> apiResponse;
public void setName(String name) {
this.name = name;
}
@ -42,4 +48,11 @@ public class ApiResponseResponse extends BaseResponse {
public void setType(String type) {
this.type = type;
}
public void addApiResponse(ApiResponseResponse childApiResponse) {
if(this.apiResponse == null) {
this.apiResponse = new HashSet<ApiResponseResponse>();
}
this.apiResponse.add(childApiResponse);
}
}

167
plugins/api/discovery/src/org/apache/cloudstack/discovery/ApiDiscoveryServiceImpl.java
View File

@ -16,25 +16,14 @@
// under the License.
package org.apache.cloudstack.discovery;
import java.lang.reflect.Field;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import javax.annotation.PostConstruct;
import javax.ejb.Local;
import javax.inject.Inject;
import com.cloud.serializer.Param;
import com.cloud.user.User;
import com.cloud.utils.ReflectUtil;
import com.cloud.utils.StringUtils;
import com.cloud.utils.component.PluggableService;
import com.google.gson.annotations.SerializedName;
import org.apache.cloudstack.acl.APIChecker;
import org.apache.cloudstack.api.APICommand;
import org.apache.cloudstack.api.BaseAsyncCmd;
import org.apache.cloudstack.api.BaseAsyncCreateCmd;
import org.apache.cloudstack.api.BaseCmd;
import org.apache.cloudstack.api.BaseResponse;
import org.apache.cloudstack.api.Parameter;
import org.apache.cloudstack.api.*;
import org.apache.cloudstack.api.command.user.discovery.ListApisCmd;
import org.apache.cloudstack.api.response.ApiDiscoveryResponse;
import org.apache.cloudstack.api.response.ApiParameterResponse;
@ -43,12 +32,11 @@ import org.apache.cloudstack.api.response.ListResponse;
import org.apache.log4j.Logger;
import org.springframework.stereotype.Component;
import com.cloud.serializer.Param;
import com.cloud.user.User;
import com.cloud.utils.ReflectUtil;
import com.cloud.utils.StringUtils;
import com.cloud.utils.component.PluggableService;
import com.google.gson.annotations.SerializedName;
import javax.annotation.PostConstruct;
import javax.ejb.Local;
import javax.inject.Inject;
import java.lang.reflect.Field;
import java.util.*;
@Component
@Local(value = ApiDiscoveryService.class)
@ -69,9 +57,9 @@ public class ApiDiscoveryServiceImpl implements ApiDiscoveryService {
long startTime = System.nanoTime();
s_apiNameDiscoveryResponseMap = new HashMap<String, ApiDiscoveryResponse>();
Set<Class<?>> cmdClasses = new HashSet<Class<?>>();
for(PluggableService service: _services) {
for(PluggableService service: _services) {
s_logger.debug(String.format("getting api commands of service: %s", service.getClass().getName()));
cmdClasses.addAll(service.getCommands());
cmdClasses.addAll(service.getCommands());
}
cmdClasses.addAll(this.getCommands());
cacheResponseMap(cmdClasses);
@ -80,72 +68,39 @@ public class ApiDiscoveryServiceImpl implements ApiDiscoveryService {
}
}
protected void cacheResponseMap(Set<Class<?>> cmdClasses) {
protected Map<String, List<String>> cacheResponseMap(Set<Class<?>> cmdClasses) {
Map<String, List<String>> responseApiNameListMap = new HashMap<String, List<String>>();
for(Class<?> cmdClass: cmdClasses) {
APICommand apiCmdAnnotation = cmdClass.getAnnotation(APICommand.class);
if (apiCmdAnnotation == null)
if (apiCmdAnnotation == null) {
apiCmdAnnotation = cmdClass.getSuperclass().getAnnotation(APICommand.class);
}
if (apiCmdAnnotation == null
|| !apiCmdAnnotation.includeInApiDoc()
|| apiCmdAnnotation.name().isEmpty())
|| apiCmdAnnotation.name().isEmpty()) {
continue;
}
String apiName = apiCmdAnnotation.name();
ApiDiscoveryResponse response = getCmdRequestMap(cmdClass, apiCmdAnnotation);
String responseName = apiCmdAnnotation.responseObject().getName();
if (!responseName.contains("SuccessResponse")) {
if (!responseApiNameListMap.containsKey(responseName))
if (!responseApiNameListMap.containsKey(responseName)) {
responseApiNameListMap.put(responseName, new ArrayList<String>());
}
responseApiNameListMap.get(responseName).add(apiName);
}
ApiDiscoveryResponse response = new ApiDiscoveryResponse();
response.setName(apiName);
response.setDescription(apiCmdAnnotation.description());
if (!apiCmdAnnotation.since().isEmpty())
response.setSince(apiCmdAnnotation.since());
response.setRelated(responseName);
Field[] responseFields = apiCmdAnnotation.responseObject().getDeclaredFields();
for(Field responseField: responseFields) {
SerializedName serializedName = responseField.getAnnotation(SerializedName.class);
if(serializedName != null) {
ApiResponseResponse responseResponse = new ApiResponseResponse();
responseResponse.setName(serializedName.value());
Param param = responseField.getAnnotation(Param.class);
if (param != null)
responseResponse.setDescription(param.description());
responseResponse.setType(responseField.getType().getSimpleName().toLowerCase());
response.addApiResponse(responseResponse);
}
ApiResponseResponse responseResponse = getFieldResponseMap(responseField);
response.addApiResponse(responseResponse);
}
Set<Field> fields = ReflectUtil.getAllFieldsForClass(cmdClass,
new Class<?>[]{BaseCmd.class, BaseAsyncCmd.class, BaseAsyncCreateCmd.class});
boolean isAsync = ReflectUtil.isCmdClassAsync(cmdClass,
new Class<?>[] {BaseAsyncCmd.class, BaseAsyncCreateCmd.class});
response.setAsync(isAsync);
for(Field field: fields) {
Parameter parameterAnnotation = field.getAnnotation(Parameter.class);
if (parameterAnnotation != null
&& parameterAnnotation.expose()
&& parameterAnnotation.includeInApiDoc()) {
ApiParameterResponse paramResponse = new ApiParameterResponse();
paramResponse.setName(parameterAnnotation.name());
paramResponse.setDescription(parameterAnnotation.description());
paramResponse.setType(parameterAnnotation.type().toString().toLowerCase());
paramResponse.setLength(parameterAnnotation.length());
paramResponse.setRequired(parameterAnnotation.required());
if (!parameterAnnotation.since().isEmpty())
paramResponse.setSince(parameterAnnotation.since());
paramResponse.setRelated(parameterAnnotation.entityType()[0].getName());
response.addParam(paramResponse);
}
}
response.setObjectName("api");
s_apiNameDiscoveryResponseMap.put(apiName, response);
}
@ -173,6 +128,76 @@ public class ApiDiscoveryServiceImpl implements ApiDiscoveryService {
}
s_apiNameDiscoveryResponseMap.put(apiName, response);
}
return responseApiNameListMap;
}
private ApiResponseResponse getFieldResponseMap(Field responseField) {
ApiResponseResponse responseResponse = new ApiResponseResponse();
SerializedName serializedName = responseField.getAnnotation(SerializedName.class);
Param param = responseField.getAnnotation(Param.class);
if (serializedName != null && param != null) {
responseResponse.setName(serializedName.value());
responseResponse.setDescription(param.description());
responseResponse.setType(responseField.getType().getSimpleName().toLowerCase());
//If response is not of primitive type - we have a nested entity
Class fieldClass = param.responseObject();
if (fieldClass != null) {
Class<?> superClass = fieldClass.getSuperclass();
if (superClass != null) {
String superName = superClass.getName();
if (superName.equals(BaseResponse.class.getName())) {
Field[] fields = fieldClass.getDeclaredFields();
for (Field field : fields) {
ApiResponseResponse innerResponse = getFieldResponseMap(field);
if (innerResponse != null) {
responseResponse.addApiResponse(innerResponse);
}
}
}
}
}
}
return responseResponse;
}
private ApiDiscoveryResponse getCmdRequestMap(Class<?> cmdClass, APICommand apiCmdAnnotation) {
String apiName = apiCmdAnnotation.name();
ApiDiscoveryResponse response = new ApiDiscoveryResponse();
response.setName(apiName);
response.setDescription(apiCmdAnnotation.description());
if (!apiCmdAnnotation.since().isEmpty()) {
response.setSince(apiCmdAnnotation.since());
}
Set<Field> fields = ReflectUtil.getAllFieldsForClass(cmdClass,
new Class<?>[]{BaseCmd.class, BaseAsyncCmd.class, BaseAsyncCreateCmd.class});
boolean isAsync = ReflectUtil.isCmdClassAsync(cmdClass,
new Class<?>[]{BaseAsyncCmd.class, BaseAsyncCreateCmd.class});
response.setAsync(isAsync);
for(Field field: fields) {
Parameter parameterAnnotation = field.getAnnotation(Parameter.class);
if (parameterAnnotation != null
&& parameterAnnotation.expose()
&& parameterAnnotation.includeInApiDoc()) {
ApiParameterResponse paramResponse = new ApiParameterResponse();
paramResponse.setName(parameterAnnotation.name());
paramResponse.setDescription(parameterAnnotation.description());
paramResponse.setType(parameterAnnotation.type().toString().toLowerCase());
paramResponse.setLength(parameterAnnotation.length());
paramResponse.setRequired(parameterAnnotation.required());
if (!parameterAnnotation.since().isEmpty()) {
paramResponse.setSince(parameterAnnotation.since());
}
paramResponse.setRelated(parameterAnnotation.entityType()[0].getName());
response.addParam(paramResponse);
}
}
return response;
}
@Override

32
server/src/com/cloud/api/response/ApiResponseSerializer.java
View File

@ -16,33 +16,25 @@
// under the License.
package com.cloud.api.response;
import java.lang.reflect.Field;
import java.lang.reflect.Method;
import java.lang.reflect.Modifier;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Date;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.apache.cloudstack.api.response.ListResponse;
import org.apache.cloudstack.api.response.*;
import org.apache.log4j.Logger;
import org.apache.cloudstack.api.ApiConstants;
import com.cloud.api.ApiDBUtils;
import com.cloud.api.ApiResponseGsonHelper;
import com.cloud.api.ApiServer;
import org.apache.cloudstack.api.BaseCmd;
import org.apache.cloudstack.api.ResponseObject;
import com.cloud.utils.encoding.URLEncoder;
import com.cloud.utils.exception.CloudRuntimeException;
import com.cloud.uuididentity.dao.IdentityDao;
import com.cloud.uuididentity.dao.IdentityDaoImpl;
import com.google.gson.Gson;
import com.google.gson.annotations.SerializedName;
import org.apache.cloudstack.api.ApiConstants;
import org.apache.cloudstack.api.BaseCmd;
import org.apache.cloudstack.api.ResponseObject;
import org.apache.cloudstack.api.response.*;
import org.apache.log4j.Logger;
import java.lang.reflect.Field;
import java.lang.reflect.Method;
import java.lang.reflect.Modifier;
import java.util.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
public class ApiResponseSerializer {
private static final Logger s_logger = Logger.getLogger(ApiResponseSerializer.class.getName());

40
server/src/com/cloud/api/response/EmptyFieldExclusionStrategy.java Normal file
View File

@ -0,0 +1,40 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package com.cloud.api.response;
import com.cloud.serializer.Param;
import com.google.gson.ExclusionStrategy;
import com.google.gson.FieldAttributes;
public class EmptyFieldExclusionStrategy implements ExclusionStrategy {
@Override
public boolean shouldSkipField(FieldAttributes fieldAttributes) {
if (fieldAttributes.getAnnotation(Param.class) != null) {
return true;
}
return false;
}
@Override
public boolean shouldSkipClass(Class<?> aClass) {
return false;
}
}

17
test/integration/smoke/test_vm_life_cycle.py
View File

@ -262,11 +262,26 @@ class TestDeployVM(cloudstackTestCase):
self.assertIsNotNone(router.publicip, msg="Router has no public ip")
self.assertIsNotNone(router.guestipaddress, msg="Router has no guest ip")
@attr(hypervisor = ["simulator"])
@attr(mode = ["basic"])
def test_basicZoneVirtualRouter(self):
"""
Tests for basic zone virtual router
1. Is Running
2. is in the account the VM was deployed in
@return:
"""
routers = list_routers(self.apiclient, account=self.account.account.name)
self.assertTrue(len(routers) > 0, msg = "No virtual router found")
router = routers[0]
self.assertEqual(router.state, 'Running', msg="Router is not in running state")
self.assertEqual(router.account, self.account.account.name, msg="Router does not belong to the account")
def tearDown(self):
pass
class TestVMLifeCycle(cloudstackTestCase):
@classmethod

15
tools/marvin/marvin/cloudstackTestClient.py
View File

@ -24,7 +24,8 @@ import string
import hashlib
class cloudstackTestClient(object):
def __init__(self, mgtSvr=None, port=8096, apiKey = None, securityKey = None, asyncTimeout=3600, defaultWorkerThreads=10, logging=None):
def __init__(self, mgtSvr=None, port=8096, apiKey = None, securityKey = None, asyncTimeout=3600,
defaultWorkerThreads=10, logging=None):
self.connection = cloudstackConnection.cloudConnection(mgtSvr, port, apiKey, securityKey, asyncTimeout, logging)
self.apiClient = cloudstackAPIClient.CloudStackAPIClient(self.connection)
self.dbConnection = None
@ -32,7 +33,6 @@ class cloudstackTestClient(object):
self.ssh = None
self.defaultWorkerThreads = defaultWorkerThreads
def dbConfigure(self, host="localhost", port=3306, user='cloud', passwd='cloud', db='cloud'):
self.dbConnection = dbConnection.dbConnection(host, port, user, passwd, db)
@ -147,7 +147,16 @@ class cloudstackTestClient(object):
if hasattr(self, "userApiClient"):
return self.userApiClient
return None
def synchronize(self):
"""
synchronize the api from an endpoint
"""
apiclient = self.getApiClient()
cmd = listApis.listApisCmd()
response = apiclient.listApis(cmd)
'''FixME, httplib has issue if more than one thread submitted'''
def submitCmdsAndWait(self, cmds, workers=1):
if self.asyncJobMgr is None:

96
tools/marvin/marvin/codegenerator.py
View File

@ -16,10 +16,12 @@
# under the License.
import xml.dom.minidom
import json
from optparse import OptionParser
from textwrap import dedent
import os
import sys
class cmdParameterProperty(object):
def __init__(self):
self.name = None
@ -97,6 +99,7 @@ class codeGenerator:
subclass += self.space + self.space + 'self.%s = None\n'%pro.name
self.subclass.append(subclass)
def generate(self, cmd):
self.cmd = cmd
@ -159,8 +162,7 @@ class codeGenerator:
fp.close()
self.code = ""
self.subclass = []
def finalize(self):
'''generate an api call'''
@ -215,8 +217,7 @@ class codeGenerator:
fp.write(basecmd)
fp.close()
def constructResponse(self, response):
def constructResponseFromXML(self, response):
paramProperty = cmdParameterProperty()
paramProperty.name = getText(response.getElementsByTagName('name'))
paramProperty.desc = getText(response.getElementsByTagName('description'))
@ -224,7 +225,23 @@ class codeGenerator:
'''This is a list'''
paramProperty.name = paramProperty.name.split('(*)')[0]
for subresponse in response.getElementsByTagName('arguments')[0].getElementsByTagName('arg'):
subProperty = self.constructResponse(subresponse)
subProperty = self.constructResponseFromXML(subresponse)
paramProperty.subProperties.append(subProperty)
return paramProperty
def constructResponseFromJSON(self, response):
paramProperty = cmdParameterProperty()
if response.has_key('name'):
paramProperty.name = response['name']
assert paramProperty.name
if response.has_key('description'):
paramProperty.desc = response['description']
if response.has_key('type') and response['type'] == 'list':
#Here list becomes a subproperty
paramProperty.name = paramProperty.name.split('(*)')[0]
for subresponse in response.getElementsByTagName('arguments')[0].getElementsByTagName('arg'):
subProperty = self.constructResponseFromXML(subresponse)
paramProperty.subProperties.append(subProperty)
return paramProperty
@ -269,18 +286,79 @@ class codeGenerator:
if response.parentNode != responseEle:
continue
paramProperty = self.constructResponse(response)
paramProperty = self.constructResponseFromXML(response)
csCmd.response.append(paramProperty)
cmds.append(csCmd)
return cmds
def generateCode(self):
def loadCmdFromJSON(self, apiStream):
if apiStream is None:
raise Exception("No APIs found through discovery")
apiDict = json.loads(apiStream)
if not apiDict.has_key('listapisresponse'):
raise Exception("API discovery plugin response failed")
if not apiDict['listapisresponse'].has_key('count'):
raise Exception("Malformed api response")
apilist = apiDict['listapisresponse']['api']
cmds = []
for cmd in apilist:
csCmd = cloudStackCmd()
if cmd.has_key('name'):
csCmd.name = cmd['name']
assert csCmd.name
if cmd.has_key('description'):
csCmd.desc = cmd['description']
if cmd.has_key('async'):
csCmd.async = cmd['isasync']
for param in cmd['params']:
paramProperty = cmdParameterProperty()
if param.has_key('name'):
paramProperty.name = param['name']
assert paramProperty.name
if param.has_key('required'):
paramProperty.required = param.getElementsByTagName('required')
if param.has_key('description'):
paramProperty.desc = param['description']
if param.has_key('type'):
paramProperty.type = param['type']
csCmd.request.append(paramProperty)
for response in cmd['response']:
paramProperty = self.constructResponseFromJSON(response)
csCmd.response.append(paramProperty)
cmds.append(csCmd)
return cmds
def generateCodeFromXML(self):
cmds = self.loadCmdFromXML()
for cmd in cmds:
self.generate(cmd)
self.finalize()
def generateCodeFromJSON(self, apiJson):
"""
Api Discovery plugin returns the supported APIs of a CloudStack endpoint.
@return: The classes in cloudstackAPI/ formed from api discovery json
"""
with open(apiJson, 'r') as apiStream:
cmds = self.loadCmdFromJSON(apiStream)
for cmd in cmds:
self.generate(cmd)
self.finalize()
def getText(elements):
return elements[0].childNodes[0].nodeValue.strip()
@ -315,5 +393,5 @@ if __name__ == "__main__":
exit(2)
cg = codeGenerator(folder, apiSpecFile)
cg.generateCode()
cg.generateCodeFromXML()

14
tools/marvin/marvin/marvinPlugin.py
View File

@ -21,6 +21,7 @@ import logging
import nose.core
from marvin.cloudstackTestCase import cloudstackTestCase
from marvin import deployDataCenter
from marvin import apiSynchronizer
from nose.plugins.base import Plugin
from functools import partial
@ -39,6 +40,10 @@ class MarvinPlugin(Plugin):
self.enableOpt = "--with-marvin"
self.logformat = logging.Formatter("%(asctime)s - %(levelname)s - %(name)s - %(message)s")
if options.sync:
self.do_sync(options.config)
return
if options.debug_log:
self.logger = logging.getLogger("NoseTestExecuteEngine")
self.debug_stream = logging.FileHandler(options.debug_log)
@ -65,6 +70,13 @@ class MarvinPlugin(Plugin):
cfg.debugLog = self.debug_stream
self.testrunner = nose.core.TextTestRunner(stream=self.result_stream, descriptions=True, verbosity=2, config=config)
def do_sync(self, config):
"""
Use the ApiDiscovery plugin exposed by the CloudStack mgmt server to rebuild the cloudStack API
"""
apiSynchronizer.sync(config)
def options(self, parser, env):
"""
@ -84,6 +96,8 @@ class MarvinPlugin(Plugin):
help="The path to the testcase debug logs [DEBUG_LOG]")
parser.add_option("--load", action="store_true", default=False, dest="load",
help="Only load the deployment configuration given")
parser.add_option("--sync", action="store_true", default=False, dest="sync",
help="Sync the APIs from the CloudStack endpoint in marvin-config")
Plugin.options(self, parser, env)