diff --git a/plugins/cpp/model/include/model/cppfunction.h b/plugins/cpp/model/include/model/cppfunction.h index 938f04aa4..7eca88a29 100644 --- a/plugins/cpp/model/include/model/cppfunction.h +++ b/plugins/cpp/model/include/model/cppfunction.h @@ -56,9 +56,6 @@ struct CppFunctionParamCountWithId #pragma db column("count(" + Parameters::id + ")") std::size_t count; - - #pragma db column(File::path) - std::string filePath; }; #pragma db view \ @@ -80,9 +77,6 @@ struct CppFunctionMcCabe #pragma db column(CppFunction::mccabe) unsigned int mccabe; - - #pragma db column(File::path) - std::string filePath; }; #pragma db view \ @@ -99,9 +93,6 @@ struct CppFunctionBumpyRoad #pragma db column(CppFunction::statementCount) unsigned int statementCount; - - #pragma db column(File::path) - std::string filePath; }; } diff --git a/plugins/cpp_metrics/model/include/model/cppcohesionmetrics.h b/plugins/cpp_metrics/model/include/model/cppcohesionmetrics.h index bc8a20629..2e4ce93eb 100644 --- a/plugins/cpp_metrics/model/include/model/cppcohesionmetrics.h +++ b/plugins/cpp_metrics/model/include/model/cppcohesionmetrics.h @@ -23,9 +23,6 @@ struct CohesionCppRecordView #pragma db column(CppEntity::astNodeId) CppAstNodeId astNodeId; - - #pragma db column(File::path) - std::string filePath; }; #pragma db view \ diff --git a/plugins/cpp_metrics/parser/include/cppmetricsparser/cppmetricsparser.h b/plugins/cpp_metrics/parser/include/cppmetricsparser/cppmetricsparser.h index 5abe1af83..574e233df 100644 --- a/plugins/cpp_metrics/parser/include/cppmetricsparser/cppmetricsparser.h +++ b/plugins/cpp_metrics/parser/include/cppmetricsparser/cppmetricsparser.h @@ -13,14 +13,43 @@ #include #include +#include #include #include +#include namespace cc { namespace parser { - + +template +class MetricsTasks +{ +public: + typedef typename std::vector::const_iterator TTaskIter; + + const TTaskIter& begin() const { return _begin; } + const TTaskIter& end() const { return _end; } + std::size_t size() const { return _size; } + + MetricsTasks( + const TTaskIter& begin_, + const TTaskIter& end_, + std::size_t size_ + ) : + _begin(begin_), + _end(end_), + _size(size_) + {} + +private: + TTaskIter _begin; + TTaskIter _end; + std::size_t _size; +}; + + class CppMetricsParser : public AbstractParser { public: @@ -41,10 +70,126 @@ class CppMetricsParser : public AbstractParser // and member functions for every type. void lackOfCohesion(); + + /// @brief Constructs an ODB query that you can use to filter only + /// the database records of the given parameter type whose path + /// is rooted under any of this parser's input paths. + /// @tparam TQueryParam The type of database records to query. + /// This type must represent an ODB view that has access to + /// (i.e. is also joined with) the File table. + /// @return A query containing the disjunction of filters. + template + odb::query getFilterPathsQuery() const + { + return cc::util::getFilterPathsQuery( + _inputPaths.begin(), _inputPaths.end()); + } + + /// @brief Calculates a metric by querying all objects of the + /// specified parameter type and passing them one-by-one to the + /// specified worker function on parallel threads. + /// This call blocks the caller thread until all workers are finished. + /// @tparam TQueryParam The type of parameters to query. + /// @param name_ The name of the metric (for progress logging). + /// @param partitions_ The number of jobs to partition the query into. + /// @param query_ A filter query for retrieving only + /// the eligible parameters for which a worker should be spawned. + /// @param worker_ The logic of the worker thread. + template + void parallelCalcMetric( + const char* name_, + std::size_t partitions_, + const odb::query& query_, + const std::function&)>& worker_) + { + typedef MetricsTasks TMetricsTasks; + typedef typename TMetricsTasks::TTaskIter TTaskIter; + typedef std::pair TJobParam; + + // Define the thread pool and job wrapper function. + LOG(info) << name_ << " : Collecting jobs from database..."; + std::unique_ptr> pool = + util::make_thread_pool(_threadCount, + [&](const TJobParam& job) + { + LOG(info) << '(' << job.first << '/' << partitions_ + << ") " << name_; + worker_(job.second); + }); + + // Cache the results of the query that will be dispatched to workers. + std::vector tasks; + util::OdbTransaction {_ctx.db} ([&, this] + { + // Storing the result directly and then calling odb::result<>::cache() + // on it does not work: odb::result<>::size() will always throw + // odb::result_not_cached. As of writing, this is a limitation of SQLite. + // So we fall back to the old-fashioned way: std::vector<> in memory. + for (const TQueryParam& param : _ctx.db->query(query_)) + tasks.emplace_back(param); + }); + + // Ensure that all workers receive at least one task. + std::size_t taskCount = tasks.size(); + if (partitions_ > taskCount) + partitions_ = taskCount; + + // Dispatch jobs to workers in discrete packets. + LOG(info) << name_ << " : Dispatching jobs on " + << _threadCount << " thread(s)..."; + std::size_t prev = 0; + TTaskIter it_prev = tasks.cbegin(); + + std::size_t i = 0; + while (i < partitions_) + { + std::size_t next = taskCount * ++i / partitions_; + std::size_t size = next - prev; + TTaskIter it_next = it_prev; + std::advance(it_next, size); + + pool->enqueue(TJobParam(i, TMetricsTasks(it_prev, it_next, size))); + + prev = next; + it_prev = it_next; + } + + // Await the termination of all workers. + pool->wait(); + LOG(info) << name_ << " : Calculation finished."; + } + + /// @brief Calculates a metric by querying all objects of the + /// specified parameter type and passing them one-by-one to the + /// specified worker function on parallel threads. + /// This call blocks the caller thread until all workers are finished. + /// @tparam TQueryParam The type of parameters to query. + /// @param name_ The name of the metric (for progress logging). + /// @param partitions_ The number of jobs to partition the query into. + /// @param worker_ The logic of the worker thread. + template + void parallelCalcMetric( + const char* name_, + std::size_t partitions_, + const std::function&)>& worker_) + { + parallelCalcMetric( + name_, + partitions_, + odb::query(), + worker_); + } + + + int _threadCount; std::vector _inputPaths; std::unordered_set _fileIdCache; std::unordered_map _astNodeIdCache; - std::unique_ptr> _pool; + + static const int functionParamsPartitionMultiplier = 5; + static const int functionMcCabePartitionMultiplier = 5; + static const int functionBumpyRoadPartitionMultiplier = 5; + static const int lackOfCohesionPartitionMultiplier = 25; }; } // parser diff --git a/plugins/cpp_metrics/parser/src/cppmetricsparser.cpp b/plugins/cpp_metrics/parser/src/cppmetricsparser.cpp index c7404d437..92561e1db 100644 --- a/plugins/cpp_metrics/parser/src/cppmetricsparser.cpp +++ b/plugins/cpp_metrics/parser/src/cppmetricsparser.cpp @@ -14,8 +14,6 @@ #include #include -#include -#include #include @@ -28,6 +26,7 @@ namespace fs = boost::filesystem; CppMetricsParser::CppMetricsParser(ParserContext& ctx_): AbstractParser(ctx_) { + _threadCount = _ctx.options["jobs"].as(); for (const std::string& path : _ctx.options["input"].as>()) _inputPaths.push_back(fs::canonical(path).string()); @@ -101,170 +100,179 @@ bool CppMetricsParser::cleanupDatabase() void CppMetricsParser::functionParameters() { - util::OdbTransaction {_ctx.db} ([&, this] + parallelCalcMetric( + "Function parameters", + _threadCount * functionParamsPartitionMultiplier,// number of jobs; adjust for granularity + getFilterPathsQuery(), + [&, this](const MetricsTasks& tasks) { - for (const model::CppFunctionParamCountWithId& paramCount - : _ctx.db->query()) + util::OdbTransaction {_ctx.db} ([&, this] { - // Skip functions that were included from external libraries. - if (!cc::util::isRootedUnderAnyOf(_inputPaths, paramCount.filePath)) - continue; - - model::CppAstNodeMetrics funcParams; - funcParams.astNodeId = paramCount.id; - funcParams.type = model::CppAstNodeMetrics::Type::PARAMETER_COUNT; - funcParams.value = paramCount.count; - _ctx.db->persist(funcParams); - } + for (const model::CppFunctionParamCountWithId& param : tasks) + { + model::CppAstNodeMetrics funcParams; + funcParams.astNodeId = param.id; + funcParams.type = model::CppAstNodeMetrics::Type::PARAMETER_COUNT; + funcParams.value = param.count; + _ctx.db->persist(funcParams); + } + }); }); } void CppMetricsParser::functionMcCabe() { - util::OdbTransaction {_ctx.db} ([&, this] + parallelCalcMetric( + "Function-level McCabe", + _threadCount * functionMcCabePartitionMultiplier,// number of jobs; adjust for granularity + getFilterPathsQuery(), + [&, this](const MetricsTasks& tasks) { - for (const model::CppFunctionMcCabe& function - : _ctx.db->query()) + util::OdbTransaction {_ctx.db} ([&, this] { - // Skip functions that were included from external libraries. - if (!cc::util::isRootedUnderAnyOf(_inputPaths, function.filePath)) - continue; - - model::CppAstNodeMetrics funcMcCabe; - funcMcCabe.astNodeId = function.astNodeId; - funcMcCabe.type = model::CppAstNodeMetrics::Type::MCCABE; - funcMcCabe.value = function.mccabe; - _ctx.db->persist(funcMcCabe); - } + for (const model::CppFunctionMcCabe& param : tasks) + { + model::CppAstNodeMetrics funcMcCabe; + funcMcCabe.astNodeId = param.astNodeId; + funcMcCabe.type = model::CppAstNodeMetrics::Type::MCCABE; + funcMcCabe.value = param.mccabe; + _ctx.db->persist(funcMcCabe); + } + }); }); } void CppMetricsParser::functionBumpyRoad() { - util::OdbTransaction {_ctx.db} ([&, this] + // Calculate the bumpy road metric for all types on parallel threads. + parallelCalcMetric( + "Bumpy road complexity", + _threadCount * functionBumpyRoadPartitionMultiplier,// number of jobs; adjust for granularity + getFilterPathsQuery(), + [&, this](const MetricsTasks& tasks) { - for (const model::CppFunctionBumpyRoad& function - : _ctx.db->query()) + util::OdbTransaction {_ctx.db} ([&, this] { - // Skip functions that were included from external libraries. - if (!cc::util::isRootedUnderAnyOf(_inputPaths, function.filePath)) - continue; - - const double dB = function.bumpiness; - const double dC = function.statementCount; - const bool empty = function.statementCount == 0; - - model::CppAstNodeMetrics metrics; - metrics.astNodeId = function.astNodeId; - metrics.type = model::CppAstNodeMetrics::Type::BUMPY_ROAD; - metrics.value = empty ? 1.0 : (dB / dC); - _ctx.db->persist(metrics); - } + for (const model::CppFunctionBumpyRoad& function : tasks) + { + const double dB = function.bumpiness; + const double dC = function.statementCount; + const bool empty = function.statementCount == 0; + + model::CppAstNodeMetrics metrics; + metrics.astNodeId = function.astNodeId; + metrics.type = model::CppAstNodeMetrics::Type::BUMPY_ROAD; + metrics.value = empty ? 1.0 : (dB / dC); + _ctx.db->persist(metrics); + } + }); }); } void CppMetricsParser::lackOfCohesion() { - util::OdbTransaction {_ctx.db} ([&, this] + // Calculate the cohesion metric for all types on parallel threads. + parallelCalcMetric( + "Lack of cohesion", + _threadCount * lackOfCohesionPartitionMultiplier, // number of jobs; adjust for granularity + getFilterPathsQuery(), + [&, this](const MetricsTasks& tasks) { - // Simplify some type names for readability. - typedef std::uint64_t HashType; - - typedef odb::query::query_columns QField; - const auto& QFieldTypeHash = QField::CppMemberType::typeHash; - - typedef odb::query::query_columns QMethod; - const auto& QMethodTypeHash = QMethod::CppMemberType::typeHash; - - typedef odb::query::query_columns QNode; - const auto& QNodeFilePath = QNode::File::path; - const auto& QNodeRange = QNode::CppAstNode::location.range; - - // Calculate the cohesion metric for all types. - for (const model::CohesionCppRecordView& type - : _ctx.db->query()) + util::OdbTransaction {_ctx.db} ([&, this] { - // Skip types that were included from external libraries. - if (!cc::util::isRootedUnderAnyOf(_inputPaths, type.filePath)) - continue; - - std::unordered_set fieldHashes; - // Query all fields of the current type. - for (const model::CohesionCppFieldView& field - : _ctx.db->query( - QFieldTypeHash == type.entityHash - )) - { - // Record these fields for later use. - fieldHashes.insert(field.entityHash); - } - std::size_t fieldCount = fieldHashes.size(); - - std::size_t methodCount = 0; - std::size_t totalCohesion = 0; - // Query all methods of the current type. - for (const model::CohesionCppMethodView& method - : _ctx.db->query( - QMethodTypeHash == type.entityHash - )) + // Simplify some type names for readability. + typedef std::uint64_t HashType; + + typedef odb::query::query_columns QField; + const auto& QFieldTypeHash = QField::CppMemberType::typeHash; + + typedef odb::query::query_columns QMethod; + const auto& QMethodTypeHash = QMethod::CppMemberType::typeHash; + + typedef odb::query::query_columns QNode; + const auto& QNodeFilePath = QNode::File::path; + const auto& QNodeRange = QNode::CppAstNode::location.range; + + for (const model::CohesionCppRecordView& type : tasks) { - // Do not consider methods with no explicit bodies. - const model::Position start(method.startLine, method.startColumn); - const model::Position end(method.endLine, method.endColumn); - if (start < end) + std::unordered_set fieldHashes; + // Query all fields of the current type. + for (const model::CohesionCppFieldView& field + : _ctx.db->query( + QFieldTypeHash == type.entityHash + )) { - std::unordered_set usedFields; - - // Query all AST nodes that use a variable for reading or writing... - for (const model::CohesionCppAstNodeView& node - : _ctx.db->query( - // ... in the same file as the current method - (QNodeFilePath == method.filePath && - // ... within the textual scope of the current method's body. - (QNodeRange.start.line >= start.line - || (QNodeRange.start.line == start.line - && QNodeRange.start.column >= start.column)) && - (QNodeRange.end.line <= end.line - || (QNodeRange.end.line == end.line - && QNodeRange.end.column <= end.column))) - )) + // Record these fields for later use. + fieldHashes.insert(field.entityHash); + } + std::size_t fieldCount = fieldHashes.size(); + + std::size_t methodCount = 0; + std::size_t totalCohesion = 0; + // Query all methods of the current type. + for (const model::CohesionCppMethodView& method + : _ctx.db->query( + QMethodTypeHash == type.entityHash + )) + { + // Do not consider methods with no explicit bodies. + const model::Position start(method.startLine, method.startColumn); + const model::Position end(method.endLine, method.endColumn); + if (start < end) { - // If this AST node is a reference to a field of the type... - if (fieldHashes.find(node.entityHash) != fieldHashes.end()) + std::unordered_set usedFields; + + // Query AST nodes that use a variable for reading or writing... + for (const model::CohesionCppAstNodeView& node + : _ctx.db->query( + // ... in the same file as the current method + (QNodeFilePath == method.filePath && + // ... within the textual scope of the current method's body. + (QNodeRange.start.line >= start.line + || (QNodeRange.start.line == start.line + && QNodeRange.start.column >= start.column)) && + (QNodeRange.end.line <= end.line + || (QNodeRange.end.line == end.line + && QNodeRange.end.column <= end.column))) + )) { - // ... then mark it as used by this method. - usedFields.insert(node.entityHash); + // If this AST node is a reference to a field of the type... + if (fieldHashes.find(node.entityHash) != fieldHashes.end()) + { + // ... then mark it as used by this method. + usedFields.insert(node.entityHash); + } } + + ++methodCount; + totalCohesion += usedFields.size(); } - - ++methodCount; - totalCohesion += usedFields.size(); } - } - // Calculate and record metrics. - const double dF = fieldCount; - const double dM = methodCount; - const double dC = totalCohesion; - const bool trivial = fieldCount == 0 || methodCount == 0; - const bool singular = methodCount == 1; - - // Standard lack of cohesion (range: [0,1]) - model::CppAstNodeMetrics lcm; - lcm.astNodeId = type.astNodeId; - lcm.type = model::CppAstNodeMetrics::Type::LACK_OF_COHESION; - lcm.value = trivial ? 0.0 : - (1.0 - dC / (dM * dF)); - _ctx.db->persist(lcm); - - // Henderson-Sellers variant (range: [0,2]) - model::CppAstNodeMetrics lcm_hs; - lcm_hs.astNodeId = type.astNodeId; - lcm_hs.type = model::CppAstNodeMetrics::Type::LACK_OF_COHESION_HS; - lcm_hs.value = trivial ? 0.0 : singular ? NAN : - ((dM - dC / dF) / (dM - 1.0)); - _ctx.db->persist(lcm_hs); - } + // Calculate and record metrics. + const double dF = fieldCount; + const double dM = methodCount; + const double dC = totalCohesion; + const bool trivial = fieldCount == 0 || methodCount == 0; + const bool singular = methodCount == 1; + + // Standard lack of cohesion (range: [0,1]) + model::CppAstNodeMetrics lcm; + lcm.astNodeId = type.astNodeId; + lcm.type = model::CppAstNodeMetrics::Type::LACK_OF_COHESION; + lcm.value = trivial ? 0.0 : + (1.0 - dC / (dM * dF)); + _ctx.db->persist(lcm); + + // Henderson-Sellers variant (range: [0,2]) + model::CppAstNodeMetrics lcm_hs; + lcm_hs.astNodeId = type.astNodeId; + lcm_hs.type = model::CppAstNodeMetrics::Type::LACK_OF_COHESION_HS; + lcm_hs.value = trivial ? 0.0 : singular ? NAN : + ((dM - dC / dF) / (dM - 1.0)); + _ctx.db->persist(lcm_hs); + } + }); }); } diff --git a/util/include/util/dbutil.h b/util/include/util/dbutil.h index 5d1c331d5..e48ddae30 100644 --- a/util/include/util/dbutil.h +++ b/util/include/util/dbutil.h @@ -131,6 +131,33 @@ bool isSingleResult(odb::result& result_) return (it_b != it_e) && (++it_b == it_e); } +/// @brief Constructs an ODB query that you can use to filter only +/// the database records of the given parameter type whose path +/// is rooted under any of the specified filter paths. +/// @tparam TQueryParam The type of database records to query. +/// This type must represent an ODB view that has access to +/// (i.e. is also joined with) the File table. +/// @tparam TIter The iterator type of the filter paths. +/// @tparam TSentinel The type of the end of the filter paths. +/// @param begin_ The iterator referring to the first filter path. +/// @param end_ The sentinel for the end of the filter paths. +/// @return A query containing the disjunction of filters. +template +odb::query getFilterPathsQuery( + TIter begin_, + const TSentinel& end_) +{ + typedef typename odb::query::query_columns QParam; + const auto& QParamPath = QParam::File::path; + constexpr char ODBWildcard = '%'; + + assert(begin_ != end_ && "At least one filter path must be provided."); + odb::query query = QParamPath.like(*begin_ + ODBWildcard); + while (++begin_ != end_) + query = query || QParamPath.like(*begin_ + ODBWildcard); + return query; +} + } // util } // cc