cloudstack/usage/src/com/cloud/usage/UsageSanityChecker.java

// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements.  See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership.  The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// the License.  You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied.  See the License for the
// specific language governing permissions and limitations
// under the License.
package com.cloud.usage;

import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.IOException;
import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.List;

import com.cloud.utils.exception.CloudRuntimeException;
import org.apache.log4j.Logger;

import com.cloud.utils.db.TransactionLegacy;

/**
 * This class must not be used concurrently because its state changes often during
 * execution in a non synchronized way
 */
public class UsageSanityChecker {

    protected static final Logger s_logger = Logger.getLogger(UsageSanityChecker.class);
    protected static final int DEFAULT_AGGREGATION_RANGE = 1440;
    protected StringBuilder errors;
    protected List<CheckCase> checkCases;
    protected String lastCheckFile = "/usr/local/libexec/sanity-check-last-id";
    protected String lastCheckId = "";
    protected int lastId = -1;
    protected int maxId = -1;
    protected Connection conn;

    protected void reset() {
        errors = new StringBuilder();
        checkCases = new ArrayList<CheckCase>();
    }

    protected boolean checkItemCountByPstmt() throws SQLException {
        boolean checkOk = true;

        for(CheckCase check : checkCases) {
            checkOk &= checkItemCountByPstmt(check);
        }

        return checkOk;
    }

    protected boolean checkItemCountByPstmt(CheckCase checkCase) throws SQLException {
        boolean checkOk = true;
        /*
         * Check for item usage records which are created after it is removed
         */
        try (PreparedStatement pstmt = conn.prepareStatement(checkCase.sqlTemplate)) {
            if(checkCase.checkId) {
                pstmt.setInt(1, lastId);
                pstmt.setInt(2, maxId);
            }
            try(ResultSet rs = pstmt.executeQuery();) {
                if (rs.next() && (rs.getInt(1) > 0)) {
                    errors.append(String.format("Error: Found %s %s\n", rs.getInt(1), checkCase.itemName));
                    checkOk = false;
                }
            }catch (Exception e)
            {
                s_logger.error("checkItemCountByPstmt:Exception:"+e.getMessage());
                throw new CloudRuntimeException("checkItemCountByPstmt:Exception:"+e.getMessage(),e);
            }
        }
        catch (Exception e)
        {
            s_logger.error("checkItemCountByPstmt:Exception:"+e.getMessage());
            throw new CloudRuntimeException("checkItemCountByPstmt:Exception:"+e.getMessage(),e);
        }
        return checkOk;
    }

    protected void checkMaxUsage() throws SQLException {
        int aggregationRange = DEFAULT_AGGREGATION_RANGE;
        try (PreparedStatement pstmt = conn.prepareStatement(
                "SELECT value FROM `cloud`.`configuration` where name = 'usage.stats.job.aggregation.range'");)
        {
            try(ResultSet rs = pstmt.executeQuery();) {
               if (rs.next()) {
                    aggregationRange = rs.getInt(1);
                } else {
                    s_logger.debug("Failed to retrieve aggregation range. Using default : " + aggregationRange);
                }
            }catch (SQLException e) {
                s_logger.error("checkMaxUsage:Exception:"+e.getMessage());
                throw new CloudRuntimeException("checkMaxUsage:Exception:"+e.getMessage());
            }
        } catch (SQLException e) {
            s_logger.error("checkMaxUsage:Exception:"+e.getMessage());
            throw new CloudRuntimeException("checkMaxUsage:Exception:"+e.getMessage());
        }
        int aggregationHours = aggregationRange / 60;

        addCheckCase("SELECT count(*) FROM `cloud_usage`.`cloud_usage` cu where usage_type not in (4,5) and raw_usage > "
                + aggregationHours,
                "usage records with raw_usage > " + aggregationHours,
                lastCheckId);
    }

    protected void checkVmUsage() {
        addCheckCase("select count(*) from cloud_usage.cloud_usage cu inner join cloud.vm_instance vm "
                + "where vm.type = 'User' and cu.usage_type in (1 , 2) "
                + "and cu.usage_id = vm.id and cu.start_date > vm.removed ",
                "Vm usage records which are created after Vm is destroyed",
                lastCheckId);

        addCheckCase("select sum(cnt) from (select count(*) as cnt from cloud_usage.usage_vm_instance "
                + "where usage_type =1 and end_date is null group by vm_instance_id "
                + "having count(vm_instance_id) > 1) c ;",
                "duplicate running Vm entries in vm usage helper table");

        addCheckCase("select sum(cnt) from (select count(*) as cnt from cloud_usage.usage_vm_instance "
                + "where usage_type =2 and end_date is null group by vm_instance_id "
                + "having count(vm_instance_id) > 1) c ;",
                "duplicate allocated Vm entries in vm usage helper table");

        addCheckCase("select count(vm_instance_id) from cloud_usage.usage_vm_instance o "
                + "where o.end_date is null and o.usage_type=1 and not exists "
                + "(select 1 from cloud_usage.usage_vm_instance i where "
                + "i.vm_instance_id=o.vm_instance_id and usage_type=2 and i.end_date is null)",
                "running Vm entries without corresponding allocated entries in vm usage helper table");
    }

    protected void checkVolumeUsage() {
        addCheckCase("select count(*) from cloud_usage.cloud_usage cu inner join cloud.volumes v where "
                + "cu.usage_type = 6 and cu.usage_id = v.id and cu.start_date > v.removed ",
                "volume usage records which are created after volume is removed",
                lastCheckId);

        addCheckCase("select sum(cnt) from (select count(*) as cnt from cloud_usage.usage_volume "
                + "where deleted is null group by id having count(id) > 1) c;",
                "duplicate records in volume usage helper table");
    }

    protected void checkTemplateISOUsage() {
        addCheckCase("select count(*) from cloud_usage.cloud_usage cu inner join cloud.template_zone_ref tzr where "
                + "cu.usage_id = tzr.template_id and cu.zone_id = tzr.zone_id and cu.usage_type in (7,8) and cu.start_date > tzr.removed ",
                "template/ISO usage records which are created after it is removed",
                lastCheckId);
    }

    protected void checkSnapshotUsage() {
        addCheckCase("select count(*) from cloud_usage.cloud_usage cu inner join cloud.snapshots s where "
                + "cu.usage_id = s.id and cu.usage_type = 9 and cu.start_date > s.removed ",
                "snapshot usage records which are created after it is removed",
                lastCheckId);
    }

    protected void readLastCheckId(){
        try(BufferedReader reader = new BufferedReader(new FileReader(lastCheckFile));) {
            String lastIdText = null;
            lastId = -1;
            if ((reader != null) && (lastIdText = reader.readLine()) != null) {
                lastId = Integer.parseInt(lastIdText);
            }
        } catch (Exception e) {
            s_logger.error("readLastCheckId:Exception:"+e.getMessage(),e);
        }
    }

    protected void readMaxId() throws SQLException {
        try (PreparedStatement pstmt = conn.prepareStatement("select max(id) from cloud_usage.cloud_usage");
             ResultSet rs = pstmt.executeQuery();)
        {
            maxId = -1;
            if (rs.next() && (rs.getInt(1) > 0)) {
                maxId = rs.getInt(1);
                lastCheckId += " and cu.id <= ?";
            }
        }catch (Exception e) {
            s_logger.error("readMaxId:"+e.getMessage(),e);
        }
    }

    protected void updateNewMaxId() {
        try (FileWriter fstream = new FileWriter(lastCheckFile);
        BufferedWriter out = new BufferedWriter(fstream);
        ){
            out.write("" + maxId);
        } catch (IOException e) {
            s_logger.error("updateNewMaxId:Exception:"+e.getMessage());
            // Error while writing last check id
        }
    }

    public String runSanityCheck() throws SQLException {

        readLastCheckId();
        if (lastId > 0) {
            lastCheckId = " and cu.id > ?";
        }

        conn = getConnection();
        readMaxId();

        reset();

        checkMaxUsage();
        checkVmUsage();
        checkVolumeUsage();
        checkTemplateISOUsage();
        checkSnapshotUsage();

        checkItemCountByPstmt();

        return errors.toString();
    }

    /**
     * Local acquisition of {@link Connection} to remove static cling
     * @return
     */
    protected Connection getConnection() {
        return TransactionLegacy.getStandaloneConnection();
    }

    public static void main(String args[]) {
        UsageSanityChecker usc = new UsageSanityChecker();
        String sanityErrors;
        try {
            sanityErrors = usc.runSanityCheck();
            if (sanityErrors.length() > 0) {
                s_logger.error(sanityErrors.toString());
            }
        } catch (SQLException e) {
            e.printStackTrace();
        }
    }

    protected void addCheckCase(String sqlTemplate, String itemName, String lastCheckId) {
        checkCases.add(new CheckCase(sqlTemplate, itemName, lastCheckId));
    }

    protected void addCheckCase(String sqlTemplate, String itemName) {
        checkCases.add(new CheckCase(sqlTemplate, itemName));
    }
}


/**
 * Just an abstraction of the kind of check to repeat across these cases
 * encapsulating what change for each specific case
 */
class CheckCase {
    public String sqlTemplate;
    public String itemName;
    public boolean checkId = false;

    public CheckCase(String sqlTemplate, String itemName, String lastCheckId) {
        checkId = true;
        this.sqlTemplate = sqlTemplate + lastCheckId;
        this.itemName = itemName;
    }

    public CheckCase(String sqlTemplate, String itemName) {
        checkId = false;
        this.sqlTemplate = sqlTemplate;
        this.itemName = itemName;
    }
}