IdentificationData.h

Go to the documentation of this file.

// --------------------------------------------------------------------------
//                   OpenMS -- Open-Source Mass Spectrometry
// --------------------------------------------------------------------------
// Copyright The OpenMS Team -- Eberhard Karls University Tuebingen,
// ETH Zurich, and Freie Universitaet Berlin 2002-2022.
//
// This software is released under a three-clause BSD license:
//  * Redistributions of source code must retain the above copyright
//    notice, this list of conditions and the following disclaimer.
//  * Redistributions in binary form must reproduce the above copyright
//    notice, this list of conditions and the following disclaimer in the
//    documentation and/or other materials provided with the distribution.
//  * Neither the name of any author or any participating institution
//    may be used to endorse or promote products derived from this software
//    without specific prior written permission.
// For a full list of authors, refer to the file AUTHORS.
// --------------------------------------------------------------------------
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
// ARE DISCLAIMED. IN NO EVENT SHALL ANY OF THE AUTHORS OR THE CONTRIBUTING
// INSTITUTIONS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
// OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
// WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
// OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
// ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
//
// --------------------------------------------------------------------------
// $Maintainer: Hendrik Weisser $
// $Authors: Hendrik Weisser $
// --------------------------------------------------------------------------

#pragma once

#include <OpenMS/METADATA/ID/ProcessingStep.h>
#include <OpenMS/METADATA/ID/Observation.h>
#include <OpenMS/METADATA/ID/DBSearchParam.h>
#include <OpenMS/METADATA/ID/IdentifiedCompound.h>
#include <OpenMS/METADATA/ID/IdentifiedSequence.h>
#include <OpenMS/METADATA/ID/InputFile.h>
#include <OpenMS/METADATA/ID/MetaData.h>
#include <OpenMS/METADATA/ID/ParentMatch.h>
#include <OpenMS/METADATA/ID/ObservationMatch.h>
#include <OpenMS/METADATA/ID/ParentSequence.h>
#include <OpenMS/METADATA/ID/ParentGroup.h>
#include <OpenMS/METADATA/ID/ObservationMatchGroup.h>
#include <OpenMS/METADATA/ID/ScoreType.h>

#include <unordered_set>

namespace OpenMS
{
  class OPENMS_DLLAPI IdentificationData: public MetaInfoInterface
  {
  public:

    // to be able to add overloads and still find the inherited ones
    using MetaInfoInterface::setMetaValue;

    // type definitions:
    using MoleculeType = IdentificationDataInternal::MoleculeType;
    using MassType = IdentificationDataInternal::MassType;

    using InputFile = IdentificationDataInternal::InputFile;
    using InputFiles = IdentificationDataInternal::InputFiles;
    using InputFileRef = IdentificationDataInternal::InputFileRef;

    using ProcessingSoftware =
      IdentificationDataInternal::ProcessingSoftware;
    using ProcessingSoftwares =
      IdentificationDataInternal::ProcessingSoftwares;
    using ProcessingSoftwareRef =
      IdentificationDataInternal::ProcessingSoftwareRef;

    using ProcessingStep = IdentificationDataInternal::ProcessingStep;
    using ProcessingSteps = IdentificationDataInternal::ProcessingSteps;
    using ProcessingStepRef = IdentificationDataInternal::ProcessingStepRef;

    using DBSearchParam = IdentificationDataInternal::DBSearchParam;
    using DBSearchParams = IdentificationDataInternal::DBSearchParams;
    using SearchParamRef = IdentificationDataInternal::SearchParamRef;
    using DBSearchSteps = IdentificationDataInternal::DBSearchSteps;

    using ScoreType = IdentificationDataInternal::ScoreType;
    using ScoreTypes = IdentificationDataInternal::ScoreTypes;
    using ScoreTypeRef = IdentificationDataInternal::ScoreTypeRef;

    using ScoredProcessingResult =
      IdentificationDataInternal::ScoredProcessingResult;

    using AppliedProcessingStep =
      IdentificationDataInternal::AppliedProcessingStep;
    using AppliedProcessingSteps =
      IdentificationDataInternal::AppliedProcessingSteps;

    using Observation = IdentificationDataInternal::Observation;
    using Observations = IdentificationDataInternal::Observations;
    using ObservationRef = IdentificationDataInternal::ObservationRef;

    using ParentSequence = IdentificationDataInternal::ParentSequence;
    using ParentSequences = IdentificationDataInternal::ParentSequences;
    using ParentSequenceRef = IdentificationDataInternal::ParentSequenceRef;

    using ParentMatch = IdentificationDataInternal::ParentMatch;
    using ParentMatches = IdentificationDataInternal::ParentMatches;

    using IdentifiedPeptide = IdentificationDataInternal::IdentifiedPeptide;
    using IdentifiedPeptides = IdentificationDataInternal::IdentifiedPeptides;
    using IdentifiedPeptideRef =
      IdentificationDataInternal::IdentifiedPeptideRef;

    using IdentifiedCompound = IdentificationDataInternal::IdentifiedCompound;
    using IdentifiedCompounds = IdentificationDataInternal::IdentifiedCompounds;
    using IdentifiedCompoundRef =
      IdentificationDataInternal::IdentifiedCompoundRef;

    using IdentifiedOligo = IdentificationDataInternal::IdentifiedOligo;
    using IdentifiedOligos = IdentificationDataInternal::IdentifiedOligos;
    using IdentifiedOligoRef = IdentificationDataInternal::IdentifiedOligoRef;

    using IdentifiedMolecule = IdentificationDataInternal::IdentifiedMolecule;

    using PeakAnnotations = IdentificationDataInternal::PeakAnnotations;

    using Adducts = IdentificationDataInternal::Adducts;
    using AdductRef = IdentificationDataInternal::AdductRef;
    using AdductOpt = IdentificationDataInternal::AdductOpt;

    using ObservationMatch = IdentificationDataInternal::ObservationMatch;
    using ObservationMatches = IdentificationDataInternal::ObservationMatches;
    using ObservationMatchRef = IdentificationDataInternal::ObservationMatchRef;

    // @TODO: allow multiple sets of groups, like with parent sequences
    // ("ParentGroupSets")?
    using ObservationMatchGroup = IdentificationDataInternal::ObservationMatchGroup;
    using ObservationMatchGroups = IdentificationDataInternal::ObservationMatchGroups;
    using MatchGroupRef = IdentificationDataInternal::MatchGroupRef;

    using ParentGroup = IdentificationDataInternal::ParentGroup;
    using ParentGroups =
      IdentificationDataInternal::ParentGroups;
    using ParentGroupRef = IdentificationDataInternal::ParentGroupRef;
    using ParentGroupSet =
      IdentificationDataInternal::ParentGroupSet;
    using ParentGroupSets =
      IdentificationDataInternal::ParentGroupSets;

    using AddressLookup = std::unordered_set<uintptr_t>;

    struct RefTranslator {
      std::map<InputFileRef, InputFileRef> input_file_refs;
      std::map<ScoreTypeRef, ScoreTypeRef> score_type_refs;
      std::map<ProcessingSoftwareRef, ProcessingSoftwareRef> processing_software_refs;
      std::map<SearchParamRef, SearchParamRef> search_param_refs;
      std::map<ProcessingStepRef, ProcessingStepRef> processing_step_refs;
      std::map<ObservationRef, ObservationRef> observation_refs;
      std::map<ParentSequenceRef, ParentSequenceRef> parent_sequence_refs;
      std::map<IdentifiedPeptideRef, IdentifiedPeptideRef> identified_peptide_refs;
      std::map<IdentifiedOligoRef, IdentifiedOligoRef> identified_oligo_refs;
      std::map<IdentifiedCompoundRef, IdentifiedCompoundRef> identified_compound_refs;
      std::map<AdductRef, AdductRef> adduct_refs;
      std::map<ObservationMatchRef, ObservationMatchRef> observation_match_refs;

      bool allow_missing = false;

      IdentifiedMolecule translate(IdentifiedMolecule old) const
      {
        switch (old.getMoleculeType())
        {
          case MoleculeType::PROTEIN:
          {
            auto pos = identified_peptide_refs.find(old.getIdentifiedPeptideRef());
            if (pos != identified_peptide_refs.end()) return pos->second;
          }
          break;
          case MoleculeType::COMPOUND:
          {
            auto pos = identified_compound_refs.find(old.getIdentifiedCompoundRef());
            if (pos != identified_compound_refs.end()) return pos->second;
          }
          break;
          case MoleculeType::RNA:
          {
            auto pos = identified_oligo_refs.find(old.getIdentifiedOligoRef());
            if (pos != identified_oligo_refs.end()) return pos->second;
          }
          break;
          default:
            throw Exception::InvalidValue(__FILE__, __LINE__, OPENMS_PRETTY_FUNCTION,
                                          "invalid molecule type",
                                          String(old.getMoleculeType()));
        }
        if (allow_missing) return old;
        throw Exception::MissingInformation(__FILE__, __LINE__, OPENMS_PRETTY_FUNCTION,
                                            "no match for reference");
      }

      ObservationMatchRef translate(ObservationMatchRef old) const
      {
        auto pos = observation_match_refs.find(old);
        if (pos != observation_match_refs.end()) return pos->second;
        if (allow_missing) return old;
        throw Exception::MissingInformation(__FILE__, __LINE__, OPENMS_PRETTY_FUNCTION,
                                            "no match for reference");
      }
    };

    IdentificationData():
      current_step_ref_(processing_steps_.end()), no_checks_(false)
    {
    }

    IdentificationData(const IdentificationData& other);

    IdentificationData(IdentificationData&& other) noexcept :
      MetaInfoInterface(std::move(other)),
      input_files_(std::move(other.input_files_)),
      processing_softwares_(std::move(other.processing_softwares_)),
      processing_steps_(std::move(other.processing_steps_)),
      db_search_params_(std::move(other.db_search_params_)),
      db_search_steps_(std::move(other.db_search_steps_)),
      score_types_(std::move(other.score_types_)),
      observations_(std::move(other.observations_)),
      parents_(std::move(other.parents_)),
      parent_groups_(std::move(other.parent_groups_)),
      identified_peptides_(std::move(other.identified_peptides_)),
      identified_compounds_(std::move(other.identified_compounds_)),
      identified_oligos_(std::move(other.identified_oligos_)),
      adducts_(std::move(other.adducts_)),
      observation_matches_(std::move(other.observation_matches_)),
      observation_match_groups_(std::move(other.observation_match_groups_)),
      current_step_ref_(std::move(other.current_step_ref_)),
      no_checks_(std::move(other.no_checks_)),
      // look-up tables:
      observation_lookup_(std::move(other.observation_lookup_)),
      parent_lookup_(std::move(other.parent_lookup_)),
      identified_peptide_lookup_(std::move(other.identified_peptide_lookup_)),
      identified_compound_lookup_(std::move(other.identified_compound_lookup_)),
      identified_oligo_lookup_(std::move(other.identified_oligo_lookup_)),
      observation_match_lookup_(std::move(other.observation_match_lookup_))
    {
    }

    InputFileRef registerInputFile(const InputFile& file);

    ProcessingSoftwareRef registerProcessingSoftware(
      const ProcessingSoftware& software);

    SearchParamRef registerDBSearchParam(const DBSearchParam& param);

    ProcessingStepRef registerProcessingStep(const ProcessingStep&
                                                 step);

    ProcessingStepRef registerProcessingStep(
      const ProcessingStep& step, SearchParamRef search_ref);

    ScoreTypeRef registerScoreType(const ScoreType& score);

    ObservationRef registerObservation(const Observation& obs);

    ParentSequenceRef registerParentSequence(const ParentSequence& parent);

    void registerParentGroupSet(const ParentGroupSet& groups);

    IdentifiedPeptideRef registerIdentifiedPeptide(const IdentifiedPeptide&
                                                   peptide);

    IdentifiedCompoundRef registerIdentifiedCompound(const IdentifiedCompound&
                                                     compound);

    IdentifiedOligoRef registerIdentifiedOligo(const IdentifiedOligo& oligo);

    AdductRef registerAdduct(const AdductInfo& adduct);

    ObservationMatchRef registerObservationMatch(const ObservationMatch& match);

    MatchGroupRef registerObservationMatchGroup(const ObservationMatchGroup& group);

    const InputFiles& getInputFiles() const
    {
      return input_files_;
    }

    const ProcessingSoftwares& getProcessingSoftwares() const
    {
      return processing_softwares_;
    }

    const ProcessingSteps& getProcessingSteps() const
    {
      return processing_steps_;
    }

    const DBSearchParams& getDBSearchParams() const
    {
      return db_search_params_;
    }

    const DBSearchSteps& getDBSearchSteps() const
    {
      return db_search_steps_;
    }

    const ScoreTypes& getScoreTypes() const
    {
      return score_types_;
    }

    const Observations& getObservations() const
    {
      return observations_;
    }

    const ParentSequences& getParentSequences() const
    {
      return parents_;
    }

    const ParentGroupSets& getParentGroupSets() const
    {
      return parent_groups_;
    }

    const IdentifiedPeptides& getIdentifiedPeptides() const
    {
      return identified_peptides_;
    }

    const IdentifiedCompounds& getIdentifiedCompounds() const
    {
      return identified_compounds_;
    }

    const IdentifiedOligos& getIdentifiedOligos() const
    {
      return identified_oligos_;
    }

    const Adducts& getAdducts() const
    {
      return adducts_;
    }

    const ObservationMatches& getObservationMatches() const
    {
      return observation_matches_;
    }

    const ObservationMatchGroups& getObservationMatchGroups() const
    {
      return observation_match_groups_;
    }

    void addScore(ObservationMatchRef match_ref, ScoreTypeRef score_ref,
                  double value);

    void setCurrentProcessingStep(ProcessingStepRef step_ref);

    ProcessingStepRef getCurrentProcessingStep();

    void clearCurrentProcessingStep();

    std::vector<ObservationMatchRef> getBestMatchPerObservation(ScoreTypeRef score_ref,
                                                                bool require_score = false) const;
    // @TODO: this currently doesn't take molecule type into account - should it?

    std::pair<ObservationMatchRef, ObservationMatchRef> getMatchesForObservation(ObservationRef obs_ref) const;

    ScoreTypeRef findScoreType(const String& score_name) const;

    void calculateCoverages(bool check_molecule_length = false);

    void cleanup(bool require_observation_match = true,
                 bool require_identified_sequence = true,
                 bool require_parent_match = true,
                 bool require_parent_group = false,
                 bool require_match_group = false);

    bool empty() const;

    RefTranslator merge(const IdentificationData& other);

    void swap(IdentificationData& other);

    void clear();

    template <class ScoredProcessingResults>
    ScoreTypeRef pickScoreType(const ScoredProcessingResults& container,
                               bool all_elements = false, bool any_score = false) const
    {
      std::map<ScoreTypeRef, Size> score_counts;

      if (any_score)
      {
        for (const auto& element : container)
        {
          for (const auto& step : element.steps_and_scores)
          {
            for (const auto& pair : step.scores)
            {
              score_counts[pair.first]++;
            }
          }
        }
      }
      else
      {
        for (const auto& element : container)
        {
          auto score_info = element.getMostRecentScore();
          if (std::get<2>(score_info)) // check success indicator
          {
            ScoreTypeRef score_ref = *std::get<1>(score_info); // unpack the option
            if (!all_elements) return score_ref;
            score_counts[score_ref]++; // elements are zero-initialized
          }
        }
      }
      if (score_counts.empty()) return score_types_.end();
      auto pos = max_element(score_counts.begin(), score_counts.end());
      // @TODO: break ties according to some criterion
      return pos->first;
    }

    void setMetaValue(const ObservationMatchRef ref, const String& key, const DataValue& value);

    void setMetaValue(const ObservationRef ref, const String& key, const DataValue& value);

    void setMetaValue(const IdentifiedMolecule& var, const String& key, const DataValue& value);

    // @TODO: add overloads for other data types derived from MetaInfoInterface

  protected:

    // containers:
    InputFiles input_files_;
    ProcessingSoftwares processing_softwares_;
    ProcessingSteps processing_steps_;
    DBSearchParams db_search_params_;
    // @TODO: store SearchParamRef inside ProcessingStep? (may not be required
    // for many processing steps)
    DBSearchSteps db_search_steps_;
    ScoreTypes score_types_;
    Observations observations_;
    ParentSequences parents_;
    ParentGroupSets parent_groups_;
    IdentifiedPeptides identified_peptides_;
    IdentifiedCompounds identified_compounds_;
    IdentifiedOligos identified_oligos_;
    Adducts adducts_;
    ObservationMatches observation_matches_;
    ObservationMatchGroups observation_match_groups_;

    ProcessingStepRef current_step_ref_;

    bool no_checks_;

    // look-up tables for fast checking of reference validity:
    AddressLookup observation_lookup_;
    AddressLookup parent_lookup_;
    // @TODO: just use one "identified_molecule_lookup_" for all molecule types?
    AddressLookup identified_peptide_lookup_;
    AddressLookup identified_compound_lookup_;
    AddressLookup identified_oligo_lookup_;
    AddressLookup observation_match_lookup_;

    void checkScoreTypes_(const std::map<ScoreTypeRef, double>& scores) const;

    void checkAppliedProcessingSteps_(const AppliedProcessingSteps&
                                      steps_and_scores) const;

    void checkParentMatches_(const ParentMatches& matches,
                             MoleculeType expected_type) const;

    void mergeScoredProcessingResults_(ScoredProcessingResult& result,
                                       const ScoredProcessingResult& other,
                                       const RefTranslator& trans);

    template <typename ElementType>
    struct ModifyMultiIndexAddProcessingStep
    {
      ModifyMultiIndexAddProcessingStep(ProcessingStepRef step_ref):
        step_ref(step_ref)
      {
      }

      void operator()(ElementType& element)
      {
        element.addProcessingStep(step_ref);
      }

      ProcessingStepRef step_ref;
    };

    template <typename ElementType>
    struct ModifyMultiIndexAddScore
    {
      ModifyMultiIndexAddScore(ScoreTypeRef score_type_ref, double value):
        score_type_ref(score_type_ref), value(value)
      {
      }

      void operator()(ElementType& element)
      {
        if (element.steps_and_scores.empty())
        {
          element.addScore(score_type_ref, value);
        }
        else // add score to most recent step
        {
          element.addScore(score_type_ref, value,
                           element.steps_and_scores.back().processing_step_opt);
        }
      }

      ScoreTypeRef score_type_ref;
      double value;
    };

    template <typename ElementType>
    struct ModifyMultiIndexRemoveParentMatches
    {
      ModifyMultiIndexRemoveParentMatches(const AddressLookup& lookup):
        lookup(lookup)
      {
      }

      void operator()(ElementType& element)
      {
        removeFromSetIf_(element.parent_matches,
                         [&](const ParentMatches::iterator it)
                         {
                           return !lookup.count(it->first);
                         });
      }

      const AddressLookup& lookup;
    };


    template <typename ContainerType, typename ElementType>
    typename ContainerType::iterator insertIntoMultiIndex_(
      ContainerType& container, const ElementType& element)
    {
      checkAppliedProcessingSteps_(element.steps_and_scores);

      auto result = container.insert(element);
      if (!result.second) // existing element - merge in new information
      {
        container.modify(result.first, [&element](ElementType& existing)
                         {
                           existing.merge(element);
                         });
      }

      // add current processing step (if necessary):
      if (current_step_ref_ != processing_steps_.end())
      {
        ModifyMultiIndexAddProcessingStep<ElementType>
          modifier(current_step_ref_);
        container.modify(result.first, modifier);
      }

      return result.first;
    }

    template <typename ContainerType, typename ElementType>
    typename ContainerType::iterator insertIntoMultiIndex_(
      ContainerType& container, const ElementType& element,
      AddressLookup& lookup)
    {
      typename ContainerType::iterator ref =
        insertIntoMultiIndex_(container, element);
      lookup.insert(uintptr_t(&(*ref)));
      return ref;
    }

    template <typename RefType, typename ContainerType>
    static bool isValidReference_(RefType ref, ContainerType& container)
    {
      for (auto it = container.begin(); it != container.end(); ++it)
      {
        if (ref == it) return true;
      }
      return false;
    }

    template <typename RefType>
    static bool isValidHashedReference_(
      RefType ref, const AddressLookup& lookup)
    {
      return lookup.count(ref);
    }

    template <typename ContainerType, typename PredicateType>
    static void removeFromSetIf_(ContainerType& container, PredicateType predicate)
    {
      for (auto it = container.begin(); it != container.end(); )
      {
        if (predicate(it))
        {
          it = container.erase(it);
        }
        else
        {
          ++it;
        }
      }
    }

    template <typename ContainerType>
    static void removeFromSetIfNotHashed_(
      ContainerType& container, const AddressLookup& lookup)
    {
      removeFromSetIf_(container, [&lookup](typename ContainerType::iterator it)
                       {
                         return !lookup.count(uintptr_t(&(*it)));
                       });
    }

    template <typename ContainerType>
    static void updateAddressLookup_(const ContainerType& container,
                                     AddressLookup& lookup)
    {
      lookup.clear();
      lookup.reserve(container.size());
      for (const auto& element : container)
      {
        lookup.insert(uintptr_t(&element));
      }
    }

    template <typename RefType, typename ContainerType>
    void setMetaValue_(const RefType ref, const String& key, const DataValue& value,
                       ContainerType& container, const AddressLookup& lookup = AddressLookup())
    {
      if (!no_checks_ && ((lookup.empty() && !isValidReference_(ref, container)) ||
                          (!lookup.empty() && !isValidHashedReference_(ref, lookup))))
      {
        String msg = "invalid reference for the given container";
        throw Exception::IllegalArgument(__FILE__, __LINE__,
                                         OPENMS_PRETTY_FUNCTION, msg);
      }
      container.modify(ref, [&key, &value](typename ContainerType::value_type& element)
      {
        element.setMetaValue(key, value);
      });
    }


    // these classes need access to manipulate data:
    friend class IDFilter;
    friend class MapAlignmentTransformer;
  };
}