diff --git a/api/src/com/cloud/exception/RequestLimitException.java b/api/src/com/cloud/exception/RequestLimitException.java new file mode 100644 index 00000000000..0142f8e8726 --- /dev/null +++ b/api/src/com/cloud/exception/RequestLimitException.java @@ -0,0 +1,43 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.exception; + +import com.cloud.utils.SerialVersionUID; +import com.cloud.utils.exception.CloudRuntimeException; + +/** + * Exception thrown if number of requests is over api rate limit set. + * @author minc + * + */ +public class RequestLimitException extends CloudRuntimeException { + + private static final long serialVersionUID = SerialVersionUID.AccountLimitException; + + protected RequestLimitException() { + super(); + } + + public RequestLimitException(String msg) { + super(msg); + } + + public RequestLimitException(String msg, Throwable cause) { + super(msg, cause); + } + +} diff --git a/api/src/org/apache/cloudstack/acl/APIChecker.java b/api/src/org/apache/cloudstack/acl/APIChecker.java index 0d0dfd1be4e..2e2b73ba782 100644 --- a/api/src/org/apache/cloudstack/acl/APIChecker.java +++ b/api/src/org/apache/cloudstack/acl/APIChecker.java @@ -17,6 +17,7 @@ package org.apache.cloudstack.acl; import com.cloud.exception.PermissionDeniedException; +import com.cloud.exception.RequestLimitException; import com.cloud.user.User; import com.cloud.utils.component.Adapter; @@ -26,5 +27,5 @@ public interface APIChecker extends Adapter { // If true, apiChecker has checked the operation // If false, apiChecker is unable to handle the operation or not implemented // On exception, checkAccess failed don't allow - boolean checkAccess(User user, String apiCommandName) throws PermissionDeniedException; + boolean checkAccess(User user, String apiCommandName) throws PermissionDeniedException, RequestLimitException; } diff --git a/api/src/org/apache/cloudstack/acl/APILimitChecker.java b/api/src/org/apache/cloudstack/acl/APILimitChecker.java new file mode 100644 index 00000000000..110742c059d --- /dev/null +++ b/api/src/org/apache/cloudstack/acl/APILimitChecker.java @@ -0,0 +1,30 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.acl; + +import org.apache.cloudstack.api.ServerApiException; + +import com.cloud.user.Account; +import com.cloud.utils.component.Adapter; + +/** + * APILimitChecker checks if we should block an API request based on pre-set account based api limit. + */ +public interface APILimitChecker extends Adapter { + // Interface for checking if the account is over its api limit + void checkLimit(Account account) throws ServerApiException; +} diff --git a/client/pom.xml b/client/pom.xml index 1bbae1f7d08..7ebe50c48f9 100644 --- a/client/pom.xml +++ b/client/pom.xml @@ -30,6 +30,11 @@ cloud-plugin-acl-static-role-based ${project.version} + + org.apache.cloudstack + cloud-plugin-api-limit-account-based + ${project.version} + org.apache.cloudstack cloud-plugin-api-discovery diff --git a/client/tomcatconf/commands.properties.in b/client/tomcatconf/commands.properties.in index 182cbd8cf3b..3740fb00633 100644 --- a/client/tomcatconf/commands.properties.in +++ b/client/tomcatconf/commands.properties.in @@ -509,3 +509,8 @@ configureSimulator=1 #### api discovery commands listApis=15 + +#### API Rate Limit service command + +getApiLimit=15 +resetApiLimit=1 diff --git a/client/tomcatconf/components.xml.in b/client/tomcatconf/components.xml.in index 958757a3563..c41d4f4f18f 100755 --- a/client/tomcatconf/components.xml.in +++ b/client/tomcatconf/components.xml.in @@ -54,6 +54,11 @@ under the License. true + + 1 + 25 + 50000 + @@ -233,6 +238,7 @@ under the License. + diff --git a/plugins/api/rate-limit/pom.xml b/plugins/api/rate-limit/pom.xml new file mode 100644 index 00000000000..1f0330916a9 --- /dev/null +++ b/plugins/api/rate-limit/pom.xml @@ -0,0 +1,51 @@ + + + 4.0.0 + cloud-plugin-api-limit-account-based + Apache CloudStack Plugin - API Rate Limit + + org.apache.cloudstack + cloudstack-plugins + 4.1.0-SNAPSHOT + ../../pom.xml + + + install + src + test + + + test/resources + + + + + org.apache.maven.plugins + maven-surefire-plugin + + -Xmx1024m + + org/apache/cloudstack/ratelimit/integration/* + + + + + + diff --git a/plugins/api/rate-limit/src/org/apache/cloudstack/api/command/admin/ratelimit/ResetApiLimitCmd.java b/plugins/api/rate-limit/src/org/apache/cloudstack/api/command/admin/ratelimit/ResetApiLimitCmd.java new file mode 100644 index 00000000000..58cab186570 --- /dev/null +++ b/plugins/api/rate-limit/src/org/apache/cloudstack/api/command/admin/ratelimit/ResetApiLimitCmd.java @@ -0,0 +1,99 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.api.command.admin.ratelimit; + +import org.apache.cloudstack.api.ACL; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.ApiErrorCode; +import org.apache.cloudstack.api.BaseCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.PlugService; +import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.api.response.AccountResponse; +import org.apache.cloudstack.api.response.ApiLimitResponse; +import org.apache.cloudstack.api.response.SuccessResponse; +import org.apache.cloudstack.ratelimit.ApiRateLimitService; +import org.apache.log4j.Logger; + +import com.cloud.user.Account; +import com.cloud.user.UserContext; + +@APICommand(name = "resetApiLimit", responseObject=ApiLimitResponse.class, description="Reset api count") +public class ResetApiLimitCmd extends BaseCmd { + private static final Logger s_logger = Logger.getLogger(ResetApiLimitCmd.class.getName()); + + private static final String s_name = "resetapilimitresponse"; + + @PlugService + ApiRateLimitService _apiLimitService; + + ///////////////////////////////////////////////////// + //////////////// API parameters ///////////////////// + ///////////////////////////////////////////////////// + + @ACL + @Parameter(name=ApiConstants.ACCOUNT, type=CommandType.UUID, entityType=AccountResponse.class, + description="the ID of the acount whose limit to be reset") + private Long accountId; + + ///////////////////////////////////////////////////// + /////////////////// Accessors /////////////////////// + ///////////////////////////////////////////////////// + + + public Long getAccountId() { + return accountId; + } + + + public void setAccountId(Long accountId) { + this.accountId = accountId; + } + + + ///////////////////////////////////////////////////// + /////////////// API Implementation/////////////////// + ///////////////////////////////////////////////////// + + + @Override + public String getCommandName() { + return s_name; + } + + @Override + public long getEntityOwnerId() { + Account account = UserContext.current().getCaller(); + if (account != null) { + return account.getId(); + } + + return Account.ACCOUNT_ID_SYSTEM; // no account info given, parent this command to SYSTEM so ERROR events are tracked + } + + @Override + public void execute(){ + boolean result = _apiLimitService.resetApiLimit(this.accountId); + if (result) { + SuccessResponse response = new SuccessResponse(getCommandName()); + this.setResponseObject(response); + } else { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to reset api limit counter"); + } + } +} diff --git a/plugins/api/rate-limit/src/org/apache/cloudstack/api/command/user/ratelimit/GetApiLimitCmd.java b/plugins/api/rate-limit/src/org/apache/cloudstack/api/command/user/ratelimit/GetApiLimitCmd.java new file mode 100644 index 00000000000..2b7b8e6dbc1 --- /dev/null +++ b/plugins/api/rate-limit/src/org/apache/cloudstack/api/command/user/ratelimit/GetApiLimitCmd.java @@ -0,0 +1,89 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.api.command.user.ratelimit; + +import java.util.ArrayList; +import java.util.List; + +import org.apache.cloudstack.api.ACL; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.BaseCmd; +import org.apache.cloudstack.api.BaseListCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.PlugService; +import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.api.BaseCmd.CommandType; +import org.apache.cloudstack.api.command.admin.ratelimit.ResetApiLimitCmd; +import org.apache.cloudstack.api.response.AccountResponse; +import org.apache.cloudstack.api.response.ApiLimitResponse; +import org.apache.cloudstack.api.response.PhysicalNetworkResponse; +import org.apache.log4j.Logger; + +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.response.ListResponse; +import org.apache.cloudstack.ratelimit.ApiRateLimitService; +import com.cloud.exception.ConcurrentOperationException; +import com.cloud.exception.InsufficientCapacityException; +import com.cloud.exception.InvalidParameterValueException; +import com.cloud.exception.ResourceAllocationException; +import com.cloud.exception.ResourceUnavailableException; +import com.cloud.user.Account; +import com.cloud.user.UserContext; +import com.cloud.utils.exception.CloudRuntimeException; + +@APICommand(name = "getApiLimit", responseObject=ApiLimitResponse.class, description="Get API limit count for the caller") +public class GetApiLimitCmd extends BaseCmd { + private static final Logger s_logger = Logger.getLogger(GetApiLimitCmd.class.getName()); + + private static final String s_name = "getapilimitresponse"; + + @PlugService + ApiRateLimitService _apiLimitService; + + + + + ///////////////////////////////////////////////////// + /////////////// API Implementation/////////////////// + ///////////////////////////////////////////////////// + + @Override + public String getCommandName() { + return s_name; + } + + @Override + public long getEntityOwnerId() { + Account account = UserContext.current().getCaller(); + if (account != null) { + return account.getId(); + } + + return Account.ACCOUNT_ID_SYSTEM; // no account info given, parent this command to SYSTEM so ERROR events are tracked + } + + @Override + public void execute(){ + Account caller = UserContext.current().getCaller(); + ApiLimitResponse response = _apiLimitService.searchApiLimit(caller); + response.setResponseName(getCommandName()); + response.setObjectName("apilimit"); + this.setResponseObject(response); + } +} + + diff --git a/plugins/api/rate-limit/src/org/apache/cloudstack/api/response/ApiLimitResponse.java b/plugins/api/rate-limit/src/org/apache/cloudstack/api/response/ApiLimitResponse.java new file mode 100644 index 00000000000..245e8f15d8a --- /dev/null +++ b/plugins/api/rate-limit/src/org/apache/cloudstack/api/response/ApiLimitResponse.java @@ -0,0 +1,82 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.api.response; + +import org.apache.cloudstack.api.ApiConstants; +import com.cloud.serializer.Param; +import com.google.gson.annotations.SerializedName; +import org.apache.cloudstack.api.BaseResponse; + + +public class ApiLimitResponse extends BaseResponse { + @SerializedName(ApiConstants.ACCOUNT_ID) @Param(description="the account uuid of the api remaining count") + private String accountId; + + @SerializedName(ApiConstants.ACCOUNT) @Param(description="the account name of the api remaining count") + private String accountName; + + @SerializedName("apiIssued") @Param(description="number of api already issued") + private int apiIssued; + + @SerializedName("apiAllowed") @Param(description="currently allowed number of apis") + private int apiAllowed; + + @SerializedName("expireAfter") @Param(description="seconds left to reset counters") + private long expireAfter; + + public void setAccountId(String accountId) { + this.accountId = accountId; + } + + public void setAccountName(String accountName) { + this.accountName = accountName; + } + + public void setApiIssued(int apiIssued) { + this.apiIssued = apiIssued; + } + + public void setApiAllowed(int apiAllowed) { + this.apiAllowed = apiAllowed; + } + + public void setExpireAfter(long duration) { + this.expireAfter = duration; + } + + public String getAccountId() { + return accountId; + } + + public String getAccountName() { + return accountName; + } + + public int getApiIssued() { + return apiIssued; + } + + public int getApiAllowed() { + return apiAllowed; + } + + public long getExpireAfter() { + return expireAfter; + } + + +} diff --git a/plugins/api/rate-limit/src/org/apache/cloudstack/ratelimit/ApiRateLimitService.java b/plugins/api/rate-limit/src/org/apache/cloudstack/ratelimit/ApiRateLimitService.java new file mode 100644 index 00000000000..c5b715019b6 --- /dev/null +++ b/plugins/api/rate-limit/src/org/apache/cloudstack/ratelimit/ApiRateLimitService.java @@ -0,0 +1,37 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.ratelimit; + +import org.apache.cloudstack.api.response.ApiLimitResponse; +import com.cloud.user.Account; +import com.cloud.utils.component.PluggableService; + +/** + * Provide API rate limit service + * @author minc + * + */ +public interface ApiRateLimitService extends PluggableService{ + + public ApiLimitResponse searchApiLimit(Account caller); + + public boolean resetApiLimit(Long accountId); + + public void setTimeToLive(int timeToLive); + + public void setMaxAllowed(int max); +} diff --git a/plugins/api/rate-limit/src/org/apache/cloudstack/ratelimit/ApiRateLimitServiceImpl.java b/plugins/api/rate-limit/src/org/apache/cloudstack/ratelimit/ApiRateLimitServiceImpl.java new file mode 100644 index 00000000000..303b92da5ed --- /dev/null +++ b/plugins/api/rate-limit/src/org/apache/cloudstack/ratelimit/ApiRateLimitServiceImpl.java @@ -0,0 +1,196 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.ratelimit; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import javax.ejb.Local; +import javax.naming.ConfigurationException; + +import net.sf.ehcache.Cache; +import net.sf.ehcache.CacheManager; + +import org.apache.log4j.Logger; + +import org.apache.cloudstack.acl.APIChecker; +import org.apache.cloudstack.api.command.admin.ratelimit.ResetApiLimitCmd; +import org.apache.cloudstack.api.command.user.ratelimit.GetApiLimitCmd; +import org.apache.cloudstack.api.response.ApiLimitResponse; + +import com.cloud.exception.PermissionDeniedException; +import com.cloud.exception.RequestLimitException; +import com.cloud.user.Account; +import com.cloud.user.AccountService; +import com.cloud.user.User; +import com.cloud.utils.component.AdapterBase; +import com.cloud.utils.component.Inject; + +@Local(value = APIChecker.class) +public class ApiRateLimitServiceImpl extends AdapterBase implements APIChecker, ApiRateLimitService { + private static final Logger s_logger = Logger.getLogger(ApiRateLimitServiceImpl.class); + + /** + * Fixed time duration where api rate limit is set, in seconds + */ + private int timeToLive = 1; + + /** + * Max number of api requests during timeToLive duration. + */ + private int maxAllowed = 30; + + private LimitStore _store = null; + + @Inject + AccountService _accountService; + + + + @Override + public boolean configure(String name, Map params) throws ConfigurationException { + super.configure(name, params); + + if (_store == null) { + // not configured yet, note that since this class is both adapter + // and pluggableService, so this method + // may be invoked twice in ComponentLocator. + // get global configured duration and max values + Object duration = params.get("api.throttling.interval"); + if (duration != null) { + timeToLive = Integer.parseInt((String) duration); + } + Object maxReqs = params.get("api.throttling.max"); + if (maxReqs != null) { + maxAllowed = Integer.parseInt((String) maxReqs); + } + // create limit store + EhcacheLimitStore cacheStore = new EhcacheLimitStore(); + int maxElements = 10000; + Object cachesize = params.get("api.throttling.cachesize"); + if ( cachesize != null ){ + maxElements = Integer.parseInt((String)cachesize); + } + CacheManager cm = CacheManager.create(); + Cache cache = new Cache("api-limit-cache", maxElements, false, false, timeToLive, timeToLive); + cm.addCache(cache); + s_logger.info("Limit Cache created with timeToLive=" + timeToLive + ", maxAllowed=" + maxAllowed + ", maxElements=" + maxElements ); + cacheStore.setCache(cache); + _store = cacheStore; + + } + + return true; + + } + + + + @Override + public ApiLimitResponse searchApiLimit(Account caller) { + ApiLimitResponse response = new ApiLimitResponse(); + response.setAccountId(caller.getUuid()); + response.setAccountName(caller.getAccountName()); + StoreEntry entry = _store.get(caller.getId()); + if (entry == null) { + + /* Populate the entry, thus unlocking any underlying mutex */ + entry = _store.create(caller.getId(), timeToLive); + response.setApiIssued(0); + response.setApiAllowed(maxAllowed); + response.setExpireAfter(timeToLive); + } + else{ + response.setApiIssued(entry.getCounter()); + response.setApiAllowed(maxAllowed - entry.getCounter()); + response.setExpireAfter(entry.getExpireDuration()); + } + + return response; + } + + + + @Override + public boolean resetApiLimit(Long accountId) { + if ( accountId != null ){ + _store.create(accountId, timeToLive); + } + else{ + _store.resetCounters(); + } + return true; + } + + + + @Override + public boolean checkAccess(User user, String apiCommandName) throws PermissionDeniedException, RequestLimitException { + Long accountId = user.getAccountId(); + Account account = _accountService.getAccount(accountId); + if ( _accountService.isRootAdmin(account.getType())){ + // no API throttling on root admin + return true; + } + StoreEntry entry = _store.get(accountId); + + if (entry == null) { + + /* Populate the entry, thus unlocking any underlying mutex */ + entry = _store.create(accountId, timeToLive); + } + + /* Increment the client count and see whether we have hit the maximum allowed clients yet. */ + int current = entry.incrementAndGet(); + + if (current <= maxAllowed) { + s_logger.trace("account (" + account.getAccountId() + "," + account.getAccountName() + ") has current count = " + current); + return true; + } else { + long expireAfter = entry.getExpireDuration(); + // for this exception, we can just show the same message to user and admin users. + String msg = "The given user has reached his/her account api limit, please retry after " + expireAfter + " ms."; + s_logger.warn(msg); + throw new RequestLimitException(msg); + } + } + + + @Override + public List> getCommands() { + List> cmdList = new ArrayList>(); + cmdList.add(ResetApiLimitCmd.class); + cmdList.add(GetApiLimitCmd.class); + return cmdList; + } + + + @Override + public void setTimeToLive(int timeToLive) { + this.timeToLive = timeToLive; + } + + + + @Override + public void setMaxAllowed(int max) { + this.maxAllowed = max; + + } + + +} diff --git a/plugins/api/rate-limit/src/org/apache/cloudstack/ratelimit/EhcacheLimitStore.java b/plugins/api/rate-limit/src/org/apache/cloudstack/ratelimit/EhcacheLimitStore.java new file mode 100644 index 00000000000..659cf81b0e6 --- /dev/null +++ b/plugins/api/rate-limit/src/org/apache/cloudstack/ratelimit/EhcacheLimitStore.java @@ -0,0 +1,99 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.ratelimit; + +import net.sf.ehcache.Ehcache; +import net.sf.ehcache.Element; +import net.sf.ehcache.constructs.blocking.BlockingCache; +import net.sf.ehcache.constructs.blocking.LockTimeoutException; + +/** + * A Limit store implementation using Ehcache. + * @author minc + * + */ +public class EhcacheLimitStore implements LimitStore { + + + private BlockingCache cache; + + + public void setCache(Ehcache cache) { + BlockingCache ref; + + if (!(cache instanceof BlockingCache)) { + ref = new BlockingCache(cache); + cache.getCacheManager().replaceCacheWithDecoratedCache(cache, new BlockingCache(cache)); + } else { + ref = (BlockingCache) cache; + } + + this.cache = ref; + } + + + @Override + public StoreEntry create(Long key, int timeToLive) { + StoreEntryImpl result = new StoreEntryImpl(timeToLive); + Element element = new Element(key, result); + element.setTimeToLive(timeToLive); + cache.put(element); + return result; + } + + @Override + public StoreEntry get(Long key) { + + Element entry = null; + + try { + + /* This may block. */ + entry = cache.get(key); + } catch (LockTimeoutException e) { + throw new RuntimeException(); + } catch (RuntimeException e) { + + /* Release the lock that may have been acquired. */ + cache.put(new Element(key, null)); + } + + StoreEntry result = null; + + if (entry != null) { + + /* + * We don't need to check isExpired() on the result, since ehcache takes care of expiring entries for us. + * c.f. the get(Key) implementation in this class. + */ + result = (StoreEntry) entry.getObjectValue(); + } + + return result; + } + + + + @Override + public void resetCounters() { + cache.removeAll(); + + } + + + +} diff --git a/plugins/api/rate-limit/src/org/apache/cloudstack/ratelimit/LimitStore.java b/plugins/api/rate-limit/src/org/apache/cloudstack/ratelimit/LimitStore.java new file mode 100644 index 00000000000..a5e086b3029 --- /dev/null +++ b/plugins/api/rate-limit/src/org/apache/cloudstack/ratelimit/LimitStore.java @@ -0,0 +1,51 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.ratelimit; + +import com.cloud.user.Account; + +/** + * Interface to define how an api limit store should work. + * @author minc + * + */ +public interface LimitStore { + + /** + * Returns a store entry for the given account. A value of null means that there is no + * such entry and the calling client must call create to avoid + * other clients potentially being blocked without any hope of progressing. A non-null + * entry means that it has not expired and can be used to determine whether the current client should be allowed to + * proceed with the rate-limited action or not. + * + */ + StoreEntry get(Long account); + + /** + * Creates a new store entry + * + * @param account + * the user account, key to the store + * @param timeToLiveInSecs + * the positive time-to-live in seconds + * @return a non-null entry + */ + StoreEntry create(Long account, int timeToLiveInSecs); + + void resetCounters(); + +} diff --git a/plugins/api/rate-limit/src/org/apache/cloudstack/ratelimit/StoreEntry.java b/plugins/api/rate-limit/src/org/apache/cloudstack/ratelimit/StoreEntry.java new file mode 100644 index 00000000000..76e8a2d9281 --- /dev/null +++ b/plugins/api/rate-limit/src/org/apache/cloudstack/ratelimit/StoreEntry.java @@ -0,0 +1,33 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.ratelimit; + +/** + * Interface for each entry in LimitStore. + * @author minc + * + */ +public interface StoreEntry { + + int getCounter(); + + int incrementAndGet(); + + boolean isExpired(); + + long getExpireDuration(); /* seconds to reset counter */ +} diff --git a/plugins/api/rate-limit/src/org/apache/cloudstack/ratelimit/StoreEntryImpl.java b/plugins/api/rate-limit/src/org/apache/cloudstack/ratelimit/StoreEntryImpl.java new file mode 100644 index 00000000000..e8143e52370 --- /dev/null +++ b/plugins/api/rate-limit/src/org/apache/cloudstack/ratelimit/StoreEntryImpl.java @@ -0,0 +1,64 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.ratelimit; + +import java.util.concurrent.atomic.AtomicInteger; + +/** + * Implementation of limit store entry. + * @author minc + * + */ +public class StoreEntryImpl implements StoreEntry { + + private final long expiry; + + private final AtomicInteger counter; + + StoreEntryImpl(int timeToLive) { + this.expiry = System.currentTimeMillis() + timeToLive * 1000; + this.counter = new AtomicInteger(0); + } + + + @Override + public boolean isExpired() { + return System.currentTimeMillis() > expiry; + } + + + + @Override + public long getExpireDuration() { + if ( isExpired() ) + return 0; // already expired + else { + return expiry - System.currentTimeMillis(); + } + } + + + @Override + public int incrementAndGet() { + return this.counter.incrementAndGet(); + } + + @Override + public int getCounter(){ + return this.counter.get(); + } +} diff --git a/plugins/api/rate-limit/test/org/apache/cloudstack/ratelimit/ApiRateLimitTest.java b/plugins/api/rate-limit/test/org/apache/cloudstack/ratelimit/ApiRateLimitTest.java new file mode 100644 index 00000000000..85eeaaf4223 --- /dev/null +++ b/plugins/api/rate-limit/test/org/apache/cloudstack/ratelimit/ApiRateLimitTest.java @@ -0,0 +1,226 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.ratelimit; + +import java.util.Collections; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + +import javax.naming.ConfigurationException; + +import org.apache.cloudstack.api.response.ApiLimitResponse; +import org.apache.cloudstack.ratelimit.ApiRateLimitServiceImpl; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +import com.cloud.exception.RequestLimitException; +import com.cloud.user.Account; +import com.cloud.user.AccountService; +import com.cloud.user.AccountVO; +import com.cloud.user.User; +import com.cloud.user.UserVO; + +import static org.junit.Assert.*; +import static org.mockito.Mockito.*; + +public class ApiRateLimitTest { + + static ApiRateLimitServiceImpl _limitService = new ApiRateLimitServiceImpl(); + static AccountService _accountService = mock(AccountService.class); + private static long acctIdSeq = 5L; + private static Account testAccount; + + @BeforeClass + public static void setUp() throws ConfigurationException { + + _limitService.configure("ApiRateLimitTest", Collections. emptyMap()); + + _limitService._accountService = _accountService; + + // Standard responses + AccountVO acct = new AccountVO(acctIdSeq); + acct.setType(Account.ACCOUNT_TYPE_NORMAL); + acct.setAccountName("demo"); + testAccount = acct; + + when(_accountService.getAccount(5L)).thenReturn(testAccount); + when(_accountService.isRootAdmin(Account.ACCOUNT_TYPE_NORMAL)).thenReturn(false); + } + + @Before + public void testSetUp() { + // reset counter for each test + _limitService.resetApiLimit(null); + } + + private User createFakeUser(){ + UserVO user = new UserVO(); + user.setAccountId(acctIdSeq); + return user; + } + + private boolean isUnderLimit(User key){ + try{ + _limitService.checkAccess(key, null); + return true; + } + catch (RequestLimitException ex){ + return false; + } + } + + @Test + public void sequentialApiAccess() { + int allowedRequests = 1; + _limitService.setMaxAllowed(allowedRequests); + _limitService.setTimeToLive(1); + + User key = createFakeUser(); + assertTrue("Allow for the first request", isUnderLimit(key)); + + assertFalse("Second request should be blocked, since we assume that the two api " + + " accesses take less than a second to perform", isUnderLimit(key)); + } + + @Test + public void canDoReasonableNumberOfApiAccessPerSecond() throws Exception { + int allowedRequests = 200; + _limitService.setMaxAllowed(allowedRequests); + _limitService.setTimeToLive(1); + + User key = createFakeUser(); + + for (int i = 0; i < allowedRequests; i++) { + assertTrue("We should allow " + allowedRequests + " requests per second, but failed at request " + i, isUnderLimit(key)); + } + + + assertFalse("We should block >" + allowedRequests + " requests per second", isUnderLimit(key)); + } + + @Test + public void multipleClientsCanAccessWithoutBlocking() throws Exception { + int allowedRequests = 200; + _limitService.setMaxAllowed(allowedRequests); + _limitService.setTimeToLive(1); + + + final User key = createFakeUser(); + + int clientCount = allowedRequests; + Runnable[] clients = new Runnable[clientCount]; + final boolean[] isUsable = new boolean[clientCount]; + + final CountDownLatch startGate = new CountDownLatch(1); + + final CountDownLatch endGate = new CountDownLatch(clientCount); + + + for (int i = 0; i < isUsable.length; ++i) { + final int j = i; + clients[j] = new Runnable() { + + /** + * {@inheritDoc} + */ + @Override + public void run() { + try { + startGate.await(); + + isUsable[j] = isUnderLimit(key); + + } catch (InterruptedException e) { + e.printStackTrace(); + } finally { + endGate.countDown(); + } + } + }; + } + + ExecutorService executor = Executors.newFixedThreadPool(clientCount); + + for (Runnable runnable : clients) { + executor.execute(runnable); + } + + startGate.countDown(); + + endGate.await(); + + for (boolean b : isUsable) { + assertTrue("Concurrent client request should be allowed within limit", b); + } + } + + @Test + public void expiryOfCounterIsSupported() throws Exception { + int allowedRequests = 1; + _limitService.setMaxAllowed(allowedRequests); + _limitService.setTimeToLive(1); + + User key = this.createFakeUser(); + + assertTrue("The first request should be allowed", isUnderLimit(key)); + + // Allow the token to expire + Thread.sleep(1001); + + assertTrue("Another request after interval should be allowed as well", isUnderLimit(key)); + } + + @Test + public void verifyResetCounters() throws Exception { + int allowedRequests = 1; + _limitService.setMaxAllowed(allowedRequests); + _limitService.setTimeToLive(1); + + User key = this.createFakeUser(); + + assertTrue("The first request should be allowed", isUnderLimit(key)); + + assertFalse("Another request should be blocked", isUnderLimit(key)); + + _limitService.resetApiLimit(key.getAccountId()); + + assertTrue("Another request should be allowed after reset counter", isUnderLimit(key)); + } + + + @Test + public void verifySearchCounter() throws Exception { + int allowedRequests = 10; + _limitService.setMaxAllowed(allowedRequests); + _limitService.setTimeToLive(1); + + User key = this.createFakeUser(); + + for ( int i = 0; i < 5; i++ ){ + assertTrue("Issued 5 requests", isUnderLimit(key)); + } + + ApiLimitResponse response = _limitService.searchApiLimit(testAccount); + assertEquals("apiIssued is incorrect", 5, response.getApiIssued()); + assertEquals("apiAllowed is incorrect", 5, response.getApiAllowed()); + assertTrue("expiredAfter is incorrect", response.getExpireAfter() < 1000); + + } + +} diff --git a/plugins/api/rate-limit/test/org/apache/cloudstack/ratelimit/integration/APITest.java b/plugins/api/rate-limit/test/org/apache/cloudstack/ratelimit/integration/APITest.java new file mode 100644 index 00000000000..7701b1515b0 --- /dev/null +++ b/plugins/api/rate-limit/test/org/apache/cloudstack/ratelimit/integration/APITest.java @@ -0,0 +1,211 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.ratelimit.integration; + +import java.io.BufferedReader; +import java.io.EOFException; +import java.io.InputStreamReader; +import java.math.BigInteger; +import java.net.HttpURLConnection; +import java.net.URL; +import java.net.URLEncoder; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.HashMap; +import java.util.Iterator; + +import org.apache.cloudstack.api.response.SuccessResponse; + +import com.cloud.api.ApiGsonHelper; +import com.cloud.utils.exception.CloudRuntimeException; +import com.google.gson.Gson; + +/** + * Base class for API Test + * + * @author Min Chen + * + */ +public abstract class APITest { + + protected String rootUrl = "http://localhost:8080/client/api"; + protected String sessionKey = null; + protected String cookieToSent = null; + + + /** + * Sending an api request through Http GET + * @param command command name + * @param params command query parameters in a HashMap + * @return http request response string + */ + protected String sendRequest(String command, HashMap params){ + try { + // Construct query string + StringBuilder sBuilder = new StringBuilder(); + sBuilder.append("command="); + sBuilder.append(command); + if ( params != null && params.size() > 0){ + Iterator keys = params.keySet().iterator(); + while (keys.hasNext()){ + String key = keys.next(); + sBuilder.append("&"); + sBuilder.append(key); + sBuilder.append("="); + sBuilder.append(URLEncoder.encode(params.get(key), "UTF-8")); + } + } + + // Construct request url + String reqUrl = rootUrl + "?" + sBuilder.toString(); + + // Send Http GET request + URL url = new URL(reqUrl); + HttpURLConnection conn = (HttpURLConnection) url.openConnection(); + conn.setRequestMethod("GET"); + + if ( !command.equals("login") && cookieToSent != null){ + // add the cookie to a request + conn.setRequestProperty("Cookie", cookieToSent); + } + conn.connect(); + + + if ( command.equals("login")){ + // if it is login call, store cookie + String headerName=null; + for (int i=1; (headerName = conn.getHeaderFieldKey(i))!=null; i++) { + if (headerName.equals("Set-Cookie")) { + String cookie = conn.getHeaderField(i); + cookie = cookie.substring(0, cookie.indexOf(";")); + String cookieName = cookie.substring(0, cookie.indexOf("=")); + String cookieValue = cookie.substring(cookie.indexOf("=") + 1, cookie.length()); + cookieToSent = cookieName + "=" + cookieValue; + } + } + } + + // Get the response + StringBuilder response = new StringBuilder(); + BufferedReader rd = new BufferedReader(new InputStreamReader(conn.getInputStream())); + String line; + try { + while ((line = rd.readLine()) != null) { + response.append(line); + } + } catch (EOFException ex) { + // ignore this exception + System.out.println("EOF exception due to java bug"); + } + rd.close(); + + + + return response.toString(); + + } catch (Exception e) { + throw new CloudRuntimeException("Problem with sending api request", e); + } + } + + protected String createMD5String(String password) { + MessageDigest md5; + try { + md5 = MessageDigest.getInstance("MD5"); + } catch (NoSuchAlgorithmException e) { + throw new CloudRuntimeException("Error", e); + } + + md5.reset(); + BigInteger pwInt = new BigInteger(1, md5.digest(password.getBytes())); + + // make sure our MD5 hash value is 32 digits long... + StringBuffer sb = new StringBuffer(); + String pwStr = pwInt.toString(16); + int padding = 32 - pwStr.length(); + for (int i = 0; i < padding; i++) { + sb.append('0'); + } + sb.append(pwStr); + return sb.toString(); + } + + + protected Object fromSerializedString(String result, Class repCls) { + try { + if (result != null && !result.isEmpty()) { + // get real content + int start; + int end; + if (repCls == LoginResponse.class || repCls == SuccessResponse.class) { + + start = result.indexOf('{', result.indexOf('{') + 1); // find + // the + // second + // { + + end = result.lastIndexOf('}', result.lastIndexOf('}') - 1); // find + // the + // second + // } + // backwards + + } else { + // get real content + start = result.indexOf('{', result.indexOf('{', result.indexOf('{') + 1) + 1); // find + // the + // third + // { + end = result.lastIndexOf('}', result.lastIndexOf('}', result.lastIndexOf('}') - 1) - 1); // find + // the + // third + // } + // backwards + } + if (start < 0 || end < 0) { + throw new CloudRuntimeException("Response format is wrong: " + result); + } + String content = result.substring(start, end + 1); + Gson gson = ApiGsonHelper.getBuilder().create(); + return gson.fromJson(content, repCls); + } + return null; + } catch (RuntimeException e) { + throw new CloudRuntimeException("Caught runtime exception when doing GSON deserialization on: " + result, e); + } + } + + /** + * Login call + * @param username user name + * @param password password (plain password, we will do MD5 hash here for you) + * @return login response string + */ + protected void login(String username, String password) + { + //String md5Psw = createMD5String(password); + // send login request + HashMap params = new HashMap(); + params.put("response", "json"); + params.put("username", username); + params.put("password", password); + String result = this.sendRequest("login", params); + LoginResponse loginResp = (LoginResponse)fromSerializedString(result, LoginResponse.class); + sessionKey = loginResp.getSessionkey(); + + } +} diff --git a/plugins/api/rate-limit/test/org/apache/cloudstack/ratelimit/integration/LoginResponse.java b/plugins/api/rate-limit/test/org/apache/cloudstack/ratelimit/integration/LoginResponse.java new file mode 100644 index 00000000000..719f39c0a5e --- /dev/null +++ b/plugins/api/rate-limit/test/org/apache/cloudstack/ratelimit/integration/LoginResponse.java @@ -0,0 +1,142 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.ratelimit.integration; + +import org.apache.cloudstack.api.BaseResponse; + +import com.cloud.serializer.Param; +import com.google.gson.annotations.SerializedName; + +/** + * Login Response object + * + * @author Min Chen + * + */ +public class LoginResponse extends BaseResponse { + + @SerializedName("timeout") + @Param(description = "session timeout period") + private String timeout; + + @SerializedName("sessionkey") + @Param(description = "login session key") + private String sessionkey; + + @SerializedName("username") + @Param(description = "login username") + private String username; + + @SerializedName("userid") + @Param(description = "login user internal uuid") + private String userid; + + @SerializedName("firstname") + @Param(description = "login user firstname") + private String firstname; + + @SerializedName("lastname") + @Param(description = "login user lastname") + private String lastname; + + @SerializedName("account") + @Param(description = "login user account type") + private String account; + + @SerializedName("domainid") + @Param(description = "login user domain id") + private String domainid; + + @SerializedName("type") + @Param(description = "login user type") + private int type; + + public String getTimeout() { + return timeout; + } + + public void setTimeout(String timeout) { + this.timeout = timeout; + } + + public String getSessionkey() { + return sessionkey; + } + + public void setSessionkey(String sessionkey) { + this.sessionkey = sessionkey; + } + + public String getUsername() { + return username; + } + + public void setUsername(String username) { + this.username = username; + } + + public String getUserid() { + return userid; + } + + public void setUserid(String userid) { + this.userid = userid; + } + + public String getFirstname() { + return firstname; + } + + public void setFirstname(String firstname) { + this.firstname = firstname; + } + + public String getLastname() { + return lastname; + } + + public void setLastname(String lastname) { + this.lastname = lastname; + } + + public String getAccount() { + return account; + } + + public void setAccount(String account) { + this.account = account; + } + + public String getDomainid() { + return domainid; + } + + public void setDomainid(String domainid) { + this.domainid = domainid; + } + + public int getType() { + return type; + } + + public void setType(int type) { + this.type = type; + } + + + +} diff --git a/plugins/api/rate-limit/test/org/apache/cloudstack/ratelimit/integration/RateLimitIntegrationTest.java b/plugins/api/rate-limit/test/org/apache/cloudstack/ratelimit/integration/RateLimitIntegrationTest.java new file mode 100644 index 00000000000..72d354c6c77 --- /dev/null +++ b/plugins/api/rate-limit/test/org/apache/cloudstack/ratelimit/integration/RateLimitIntegrationTest.java @@ -0,0 +1,214 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.ratelimit.integration; + +import static org.junit.Assert.*; + +import java.util.HashMap; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + +import org.apache.cloudstack.api.response.ApiLimitResponse; +import org.apache.cloudstack.api.response.SuccessResponse; +import org.junit.Before; +import org.junit.Test; + +import com.cloud.utils.exception.CloudRuntimeException; + + +/** + * Test fixture to do integration rate limit test. + * Currently we commented out this test suite since it requires a real MS and Db running. + * + * @author Min Chen + * + */ +public class RateLimitIntegrationTest extends APITest { + + private static int apiMax = 25; // assuming ApiRateLimitService set api.throttling.max = 25 + + @Before + public void setup(){ + // always reset count for each testcase + login("admin", "password"); + + // issue reset api limit calls + final HashMap params = new HashMap(); + params.put("response", "json"); + params.put("sessionkey", sessionKey); + String resetResult = sendRequest("resetApiLimit", params); + assertNotNull("Reset count failed!", fromSerializedString(resetResult, SuccessResponse.class)); + + } + + + @Test + public void testNoApiLimitOnRootAdmin() throws Exception { + // issue list Accounts calls + final HashMap params = new HashMap(); + params.put("response", "json"); + params.put("listAll", "true"); + params.put("sessionkey", sessionKey); + // assuming ApiRateLimitService set api.throttling.max = 25 + int clientCount = 26; + Runnable[] clients = new Runnable[clientCount]; + final boolean[] isUsable = new boolean[clientCount]; + + final CountDownLatch startGate = new CountDownLatch(1); + + final CountDownLatch endGate = new CountDownLatch(clientCount); + + + for (int i = 0; i < isUsable.length; ++i) { + final int j = i; + clients[j] = new Runnable() { + + /** + * {@inheritDoc} + */ + @Override + public void run() { + try { + startGate.await(); + + sendRequest("listAccounts", params); + + isUsable[j] = true; + + } catch (CloudRuntimeException e){ + isUsable[j] = false; + e.printStackTrace(); + } catch (InterruptedException e) { + e.printStackTrace(); + } finally { + endGate.countDown(); + } + } + }; + } + + ExecutorService executor = Executors.newFixedThreadPool(clientCount); + + for (Runnable runnable : clients) { + executor.execute(runnable); + } + + startGate.countDown(); + + endGate.await(); + + int rejectCount = 0; + for ( int i = 0; i < isUsable.length; ++i){ + if ( !isUsable[i]) + rejectCount++; + } + + assertEquals("No request should be rejected!", 0, rejectCount); + + } + + + @Test + public void testApiLimitOnUser() throws Exception { + // log in using normal user + login("demo", "password"); + // issue list Accounts calls + final HashMap params = new HashMap(); + params.put("response", "json"); + params.put("listAll", "true"); + params.put("sessionkey", sessionKey); + + int clientCount = apiMax + 1; + Runnable[] clients = new Runnable[clientCount]; + final boolean[] isUsable = new boolean[clientCount]; + + final CountDownLatch startGate = new CountDownLatch(1); + + final CountDownLatch endGate = new CountDownLatch(clientCount); + + + for (int i = 0; i < isUsable.length; ++i) { + final int j = i; + clients[j] = new Runnable() { + + /** + * {@inheritDoc} + */ + @Override + public void run() { + try { + startGate.await(); + + sendRequest("listAccounts", params); + + isUsable[j] = true; + + } catch (CloudRuntimeException e){ + isUsable[j] = false; + e.printStackTrace(); + } catch (InterruptedException e) { + e.printStackTrace(); + } finally { + endGate.countDown(); + } + } + }; + } + + ExecutorService executor = Executors.newFixedThreadPool(clientCount); + + for (Runnable runnable : clients) { + executor.execute(runnable); + } + + startGate.countDown(); + + endGate.await(); + + int rejectCount = 0; + for ( int i = 0; i < isUsable.length; ++i){ + if ( !isUsable[i]) + rejectCount++; + } + + assertEquals("Only one request should be rejected!", 1, rejectCount); + + } + + @Test + public void testGetApiLimitOnUser() throws Exception { + // log in using normal user + login("demo", "password"); + + // issue an api call + HashMap params = new HashMap(); + params.put("response", "json"); + params.put("listAll", "true"); + params.put("sessionkey", sessionKey); + sendRequest("listAccounts", params); + + // issue get api limit calls + final HashMap params2 = new HashMap(); + params2.put("response", "json"); + params2.put("sessionkey", sessionKey); + String getResult = sendRequest("getApiLimit", params2); + ApiLimitResponse getLimitResp = (ApiLimitResponse)fromSerializedString(getResult, ApiLimitResponse.class); + assertEquals("Issued api count is incorrect!", 2, getLimitResp.getApiIssued() ); // should be 2 apis issues plus this getlimit api + assertEquals("Allowed api count is incorrect!", apiMax -2, getLimitResp.getApiAllowed()); + } +} diff --git a/plugins/pom.xml b/plugins/pom.xml index a42ae2967b1..7bb60a990fb 100644 --- a/plugins/pom.xml +++ b/plugins/pom.xml @@ -32,6 +32,7 @@ test + api/rate-limit api/discovery acl/static-role-based deployment-planners/user-concentrated-pod diff --git a/server/src/com/cloud/api/ApiServer.java b/server/src/com/cloud/api/ApiServer.java index 33ae0077a79..ac1ba0a651a 100755 --- a/server/src/com/cloud/api/ApiServer.java +++ b/server/src/com/cloud/api/ApiServer.java @@ -51,6 +51,7 @@ import javax.servlet.http.HttpServletResponse; import javax.servlet.http.HttpSession; import com.cloud.utils.ReflectUtil; +import org.apache.cloudstack.acl.APILimitChecker; import org.apache.cloudstack.acl.APIChecker; import org.apache.cloudstack.acl.RoleType; import org.apache.cloudstack.api.*; @@ -118,6 +119,7 @@ import com.cloud.exception.CloudAuthenticationException; import com.cloud.exception.InsufficientCapacityException; import com.cloud.exception.InvalidParameterValueException; import com.cloud.exception.PermissionDeniedException; +import com.cloud.exception.RequestLimitException; import com.cloud.exception.ResourceAllocationException; import com.cloud.exception.ResourceUnavailableException; import com.cloud.server.ManagementServer; @@ -150,6 +152,8 @@ public class ApiServer implements HttpRequestHandler { @Inject private DomainManager _domainMgr = null; @Inject private AsyncJobManager _asyncMgr = null; + @Inject(adapter = APILimitChecker.class) + protected Adapters _apiLimitCheckers; @Inject(adapter = APIChecker.class) protected Adapters _apiAccessCheckers; @@ -382,6 +386,7 @@ public class ApiServer implements HttpRequestHandler { if (UserContext.current().getCaller().getType() != Account.ACCOUNT_TYPE_ADMIN){ // hide internal details to non-admin user for security reason errorMsg = BaseCmd.USER_ERROR_MESSAGE; + } throw new ServerApiException(ApiErrorCode.INSUFFICIENT_CAPACITY_ERROR, errorMsg, ex); } @@ -585,6 +590,7 @@ public class ApiServer implements HttpRequestHandler { // if userId not null, that mean that user is logged in if (userId != null) { User user = ApiDBUtils.findUserById(userId); + try{ checkCommandAvailable(user, commandName); } @@ -592,6 +598,10 @@ public class ApiServer implements HttpRequestHandler { s_logger.debug("The given command:" + commandName + " does not exist or it is not available for user with id:" + userId); throw new ServerApiException(ApiErrorCode.UNSUPPORTED_ACTION_ERROR, "The given command does not exist or it is not available for user"); } + catch (RequestLimitException ex){ + s_logger.debug(ex.getMessage()); + throw new ServerApiException(ApiErrorCode.API_LIMIT_EXCEED, ex.getMessage()); + } return true; } else { // check against every available command to see if the command exists or not @@ -821,6 +831,7 @@ public class ApiServer implements HttpRequestHandler { return true; } + private void checkCommandAvailable(User user, String commandName) throws PermissionDeniedException { if (user == null) { throw new PermissionDeniedException("User is null for role based API access check for command" + commandName); diff --git a/server/src/com/cloud/api/ApiServlet.java b/server/src/com/cloud/api/ApiServlet.java index 92d3137d30a..0f8924a080c 100755 --- a/server/src/com/cloud/api/ApiServlet.java +++ b/server/src/com/cloud/api/ApiServlet.java @@ -304,13 +304,11 @@ public class ApiServlet extends HttpServlet { * key mechanism updateUserContext(params, session != null ? session.getId() : null); */ - auditTrailSb.insert(0, - "(userId=" + UserContext.current().getCallerUserId() + " accountId=" + UserContext.current().getCaller().getId() + " sessionId=" + (session != null ? session.getId() : null) - + ")"); + auditTrailSb.insert(0, "(userId=" + UserContext.current().getCallerUserId() + " accountId=" + + UserContext.current().getCaller().getId() + " sessionId=" + (session != null ? session.getId() : null) + ")"); String response = _apiServer.handleRequest(params, false, responseType, auditTrailSb); writeResponse(resp, response != null ? response : "", HttpServletResponse.SC_OK, responseType); - } else { if (session != null) { try { diff --git a/server/src/com/cloud/configuration/Config.java b/server/src/com/cloud/configuration/Config.java index ce3698f7854..4ae144e6ce1 100755 --- a/server/src/com/cloud/configuration/Config.java +++ b/server/src/com/cloud/configuration/Config.java @@ -360,7 +360,6 @@ public enum Config { ConcurrentSnapshotsThresholdPerHost("Advanced", ManagementServer.class, Long.class, "concurrent.snapshots.threshold.perhost", null, "Limits number of snapshots that can be handled by the host concurrently; default is NULL - unlimited", null); - private final String _category; private final Class _componentClass; private final Class _type; diff --git a/server/test/com/cloud/api/APITest.java b/server/test/com/cloud/api/APITest.java index 69c488f5a10..0b040abc3f5 100644 --- a/server/test/com/cloud/api/APITest.java +++ b/server/test/com/cloud/api/APITest.java @@ -19,17 +19,17 @@ package com.cloud.api; import java.io.BufferedReader; import java.io.EOFException; import java.io.InputStreamReader; -import java.io.OutputStreamWriter; import java.math.BigInteger; import java.net.HttpURLConnection; import java.net.URL; -import java.net.URLConnection; import java.net.URLEncoder; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; import java.util.HashMap; import java.util.Iterator; +import org.apache.cloudstack.api.response.SuccessResponse; + import com.cloud.utils.exception.CloudRuntimeException; import com.google.gson.Gson; @@ -147,17 +147,38 @@ public abstract class APITest { protected Object fromSerializedString(String result, Class repCls) { try { if (result != null && !result.isEmpty()) { - // get real content - int start = result.indexOf('{', result.indexOf('{') + 1); // find the second { - if ( start < 0 ){ + int start; + int end; + if (repCls == LoginResponse.class || repCls == SuccessResponse.class) { + + start = result.indexOf('{', result.indexOf('{') + 1); // find + // the + // second + // { + + end = result.lastIndexOf('}', result.lastIndexOf('}') - 1); // find + // the + // second + // } + // backwards + + } else { + // get real content + start = result.indexOf('{', result.indexOf('{', result.indexOf('{') + 1) + 1); // find + // the + // third + // { + end = result.lastIndexOf('}', result.lastIndexOf('}', result.lastIndexOf('}') - 1) - 1); // find + // the + // third + // } + // backwards + } + if (start < 0 || end < 0) { throw new CloudRuntimeException("Response format is wrong: " + result); } - int end = result.lastIndexOf('}', result.lastIndexOf('}')-1); // find the second } backwards - if ( end < 0 ){ - throw new CloudRuntimeException("Response format is wrong: " + result); - } - String content = result.substring(start, end+1); + String content = result.substring(start, end + 1); Gson gson = ApiGsonHelper.getBuilder().create(); return gson.fromJson(content, repCls); } diff --git a/server/test/com/cloud/api/ListPerfTest.java b/server/test/com/cloud/api/ListPerfTest.java index eb98d9187fe..b8cb97eb8f0 100644 --- a/server/test/com/cloud/api/ListPerfTest.java +++ b/server/test/com/cloud/api/ListPerfTest.java @@ -16,11 +16,18 @@ // under the License. package com.cloud.api; +import static org.junit.Assert.*; + import java.util.HashMap; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; import org.junit.Before; import org.junit.Test; +import com.cloud.utils.exception.CloudRuntimeException; + /** * Test fixture to do performance test for list command @@ -163,6 +170,4 @@ public class ListPerfTest extends APITest { } - - } diff --git a/utils/src/com/cloud/utils/exception/CSExceptionErrorCode.java b/utils/src/com/cloud/utils/exception/CSExceptionErrorCode.java index 1a0969957a6..e794ea5d9c4 100755 --- a/utils/src/com/cloud/utils/exception/CSExceptionErrorCode.java +++ b/utils/src/com/cloud/utils/exception/CSExceptionErrorCode.java @@ -64,6 +64,7 @@ public class CSExceptionErrorCode { ExceptionErrorCodeMap.put("com.cloud.exception.UnsupportedServiceException", 4390); ExceptionErrorCodeMap.put("com.cloud.exception.VirtualMachineMigrationException", 4395); ExceptionErrorCodeMap.put("com.cloud.async.AsyncCommandQueued", 4540); + ExceptionErrorCodeMap.put("com.cloud.exception.RequestLimitException", 4545); // Have a special error code for ServerApiException when it is // thrown in a standalone manner when failing to detect any of the above