src/gui/plugins/doc/basic/file_loader.cpp

Go to the documentation of this file.

/*  $Id: file_loader.cpp 18753 2009-02-06 19:10:05Z dicuccio $
 * ===========================================================================
 *
 *                            PUBLIC DOMAIN NOTICE
 *               National Center for Biotechnology Information
 *
 *  This software/database is a "United States Government Work" under the
 *  terms of the United States Copyright Act.  It was written as part of
 *  the author's official duties as a United States Government employee and
 *  thus cannot be copyrighted.  This software/database is freely available
 *  to the public for use. The National Library of Medicine and the U.S.
 *  Government have not placed any restriction on its use or reproduction.
 *
 *  Although all reasonable efforts have been taken to ensure the accuracy
 *  and reliability of the software and data, the NLM and the U.S.
 *  Government do not and cannot warrant the performance or results that
 *  may be obtained by using this software or data. The NLM and the U.S.
 *  Government disclaim all warranties, express or implied, including
 *  warranties of performance, merchantability or fitness for any particular
 *  purpose.
 *
 *  Please cite the author in any work or product based on this material.
 *
 * ===========================================================================
 *
 * Authors:  Mike DiCuccio
 *
 * File Description:
 *    CDataPlugin_FileImport - load sequence information from a file.
 */

#include <ncbi_pch.hpp>
#include "file_loader.hpp"

#include <corelib/ncbifile.hpp>

#include <gui/core/plugin_registry.hpp>
#include <gui/core/plugin_utils.hpp>
#include <gui/core/version.hpp>
#include <gui/objutils/obj_convert.hpp>
#include <gui/objects/ProjectItem.hpp>
#include <gui/objects/PluginInfoSet.hpp>
#include <gui/objects/PluginValueConstraint.hpp>

#include <util/xregexp/regexp.hpp>
#include <util/static_map.hpp>
#include <algo/phy_tree/phy_node.hpp>
#include <objtools/readers/aln_reader.hpp>
#include <objtools/readers/rm_reader.hpp>

#include "file_type_chooser_dlg.hpp"

BEGIN_NCBI_SCOPE
USING_SCOPE(objects);


typedef pair<const char*, CDataPlugin_FileImport::EFormat> TFormatPair;
static const TFormatPair sc_FormatArray[] = {
    TFormatPair("AGP format assembly",  CDataPlugin_FileImport::eFormat_AGP),
    TFormatPair("Autodetect",           CDataPlugin_FileImport::eFormat_Autodetect),
    TFormatPair("Distance Matrix",      CDataPlugin_FileImport::eFormat_DistanceMatrix),
    TFormatPair("FASTA sequence",       CDataPlugin_FileImport::eFormat_FastA),
    TFormatPair("Five-Column Feature Table", CDataPlugin_FileImport::eFormat_FiveColumn),
    TFormatPair("GFF\\/GTF format annotations",   CDataPlugin_FileImport::eFormat_GTF),
    TFormatPair("Glimmer3 Predictions",   CDataPlugin_FileImport::eFormat_Glimmer3),
    TFormatPair("NCBI Binary ASN.1",    CDataPlugin_FileImport::eFormat_AsnBinary),
    TFormatPair("NCBI Text ASN.1",      CDataPlugin_FileImport::eFormat_AsnText),
    TFormatPair("NCBI XML",             CDataPlugin_FileImport::eFormat_XML),
    TFormatPair("Newick-format Phylogenetic Tree", CDataPlugin_FileImport::eFormat_NewickTree),
    TFormatPair("Phrap\\/ACE-format assembly",   CDataPlugin_FileImport::eFormat_Phrap),
    TFormatPair("RepeatMasker OUT", CDataPlugin_FileImport::eFormat_RepeatMasker),
    TFormatPair("SNP Markers",          CDataPlugin_FileImport::eFormat_SNPMarkers),
    TFormatPair("Table",                CDataPlugin_FileImport::eFormat_Table),
    TFormatPair("Taxplot File",         CDataPlugin_FileImport::eFormat_Taxplot),
    TFormatPair("Text alignment",       CDataPlugin_FileImport::eFormat_TextAlign),
};
typedef CStaticArrayMap<const char*, CDataPlugin_FileImport::EFormat, PCase_CStr> TFormatMap;
DEFINE_STATIC_ARRAY_MAP(TFormatMap, sc_Formats, sc_FormatArray);

// define some brief synonyms for above strings
static const TFormatPair sc_SynFormatArray[] = {
    TFormatPair("agp",       CDataPlugin_FileImport::eFormat_AGP),
    TFormatPair("asnbin",    CDataPlugin_FileImport::eFormat_AsnBinary),
    TFormatPair("asntext",   CDataPlugin_FileImport::eFormat_AsnText),
    TFormatPair("auto",      CDataPlugin_FileImport::eFormat_Autodetect),
    TFormatPair("dist",      CDataPlugin_FileImport::eFormat_DistanceMatrix),
    TFormatPair("fasta",     CDataPlugin_FileImport::eFormat_FastA),
    TFormatPair("fivecol",   CDataPlugin_FileImport::eFormat_FiveColumn),
    TFormatPair("glimmer",   CDataPlugin_FileImport::eFormat_Glimmer3),
    TFormatPair("gtf",       CDataPlugin_FileImport::eFormat_GTF),
    TFormatPair("newick",    CDataPlugin_FileImport::eFormat_NewickTree),
    TFormatPair("phrap",     CDataPlugin_FileImport::eFormat_Phrap),
    TFormatPair("rmo",       CDataPlugin_FileImport::eFormat_RepeatMasker),
    TFormatPair("smk",       CDataPlugin_FileImport::eFormat_SNPMarkers),
    TFormatPair("table",     CDataPlugin_FileImport::eFormat_Table),
    TFormatPair("taxplot",   CDataPlugin_FileImport::eFormat_Taxplot),
    TFormatPair("textalign", CDataPlugin_FileImport::eFormat_TextAlign),
    TFormatPair("xml",       CDataPlugin_FileImport::eFormat_XML)
};
DEFINE_STATIC_ARRAY_MAP(TFormatMap, sc_SynFormats, sc_SynFormatArray);


/*static bool s_IsInputAlignment( CNcbiIstream& input )
{
    const streamsize ALN_TEST_MAX = 1024*1024;

    try {
        CNcbiIfstream& infile = dynamic_cast< CNcbiIfstream& >( input );
        streamsize initPos = infile.tellg();
        infile.seekg( 0, ios_base::end );
        streamsize show = infile.tellg();
        if ( ALN_TEST_MAX < infile.tellg() ) {
            //
            //  the test that follows is too expensive for files that big
            //
            infile.seekg( initPos );
            return false;
        }

        infile.seekg( 0 );
        CAlnReader reader( infile );
        // Protein is a superset of nucleotide; should work for both
        reader.SetAlphabet(CAlnReader::eAlpha_Protein);
        // clustal, paup, phylip, and fast + gaps all use '-'
        reader.SetAllGap("-");
        try {
            reader.Read(true);
        }
        catch( ... ) {
            infile.clear();
            infile.seekg( initPos );
            return false;
        }
        infile.seekg( initPos );
        return true;
    }
    catch( ... ) {
        input.clear();
        return false;
    }
};*/


static CDataPlugin_FileImport::EFormat s_GetFormatFromString(const string& fmt_str)
{
    CDataPlugin_FileImport::EFormat fmt = CDataPlugin_FileImport::eFormat_Unknown;
    TFormatMap::const_iterator iter = sc_Formats.find(fmt_str.c_str());
    if (iter != sc_Formats.end()) {
        fmt = iter->second;
    } else {
        iter = sc_SynFormats.find(fmt_str.c_str());
        if (iter != sc_SynFormats.end()) {
            fmt = iter->second;
        }
    }
    return fmt;
}

static string
s_GetStringFromFormat(CDataPlugin_FileImport::EFormat fmt)
{
    ITERATE (TFormatMap, iter, sc_Formats) {
        if (iter->second == fmt) {
            return string(iter->first);
        }
    }

    ITERATE (TFormatMap, iter, sc_SynFormats) {
        if (iter->second == fmt) {
            return string(iter->first);
        }
    }

    return string();
}



static void s_AddDefaultArguments(CPluginCommand& args)
{
    args.AddArgument("file", "File name", CPluginArg::eInputFile);
    args.AddDefaultArgument("fmt", "Format",
                            CPluginArg::eString,
                            s_GetStringFromFormat(CDataPlugin_FileImport::eFormat_Autodetect));

    {{
        CRef<CPluginValueConstraint> cons(CPluginValueConstraint::CreateSet());
        ITERATE (TFormatMap, iter, sc_Formats) {
            *cons, string(iter->first);
        }
        args.SetConstraint("fmt", *cons);
    }}
}


void CDataPlugin_FileImport::GetInfo(CPluginInfoSet& info_set)
{
    {{
        CPluginInfo& info =
            info_set.CreateInfo(CPluginVersion::eMajor, CPluginVersion::eMinor, 0,
                                string(__DATE__) + " " + string(__TIME__),
                                "CDataPlugin_FileImport",
                                "File", "Load information from a file", "");
        info.SetAffinity(CPluginInfo::eAffinity_foreground);

        // command info
        CPluginCommand& load_args = info.SetDataCommand(CPluginCommand::eCommand_load);
        s_AddDefaultArguments(load_args);
    }}


    {{
        CPluginInfo& info =
            info_set.CreateInfo(CPluginVersion::eMajor, CPluginVersion::eMinor, 0,
                                string(__DATE__) + " " + string(__TIME__),
                                "CDataPlugin_FileImport",
                                "File", "Import information from a file", "");
        info.SetAffinity(CPluginInfo::eAffinity_foreground);

        CPluginCommand& import_args = info.SetDataCommand(CPluginCommand::eCommand_import);
        import_args.AddArgument("project", "Project to Import Into", CPluginArg::eProject);
        s_AddDefaultArguments(import_args);
    }}

}


CDataPlugin_FileImport::CDataPlugin_FileImport()
{
}


CDataPlugin_FileImport::~CDataPlugin_FileImport()
{
}


// load objects from a file into an existing project
void CDataPlugin_FileImport::Import(CPluginMessage& msg)
{
    x_Load(msg);
}


// load objects from a file and request creation of a new project
void CDataPlugin_FileImport::Load(CPluginMessage& msg)
{
    x_Load(msg);
}


void CDataPlugin_FileImport::x_Load(CPluginMessage& msg)
{
    const CPluginCommand& args = msg.GetRequest().GetCommand();
    CPluginReply& reply = msg.SetReply();

    const CGBProjectHandle* project = NULL;
    if (args.HasArgument("project")) {
        project = args["project"].GetProject();
    }

    string fname = args["file"].AsString();
    if (CFile(fname).Exists()) {
        // adjust 'fmt' based on user preferences
        EFormat fmt = eFormat_Unknown;
        {{
            string fmt_str = args["fmt"].AsString();
            fmt = s_GetFormatFromString(fmt_str);
        }}
        if (fmt == eFormat_Unknown  ||  fmt == eFormat_Autodetect) {
            CNcbiIfstream istr(fname.c_str(), ios::binary);
            fmt = x_GetFormat(istr);
        }

        if (fmt == eFormat_Unknown) {
            CFileTypeChooserDlg dlg;
            ITERATE (TFormatMap, iter, sc_Formats) {
                if (iter->second == eFormat_Autodetect) {
                    continue;
                }
                dlg.AddType(iter->first);
            }
            dlg.CenterOnActive();
            if (dlg.ShowModal() != eOK) {
                reply.SetStatus(eMessageStatus_ignored);
                return;
            }

            string type_str = dlg.GetType();
            ITERATE (TFormatMap, iter, sc_Formats) {
                if (iter->first == type_str) {
                    fmt = iter->second;
                    break;
                }
            }

            if (fmt == eFormat_Unknown) {
                reply.AddAction(CPluginReplyAction::e_Error,
                                "The type you selected was not understood");
                return;
            }
        }

        CRef<CPluginMessage> msg;
        CPluginUtils::TDispatchFlags disp_flags = CPluginUtils::fDispatchDefaults;

        switch (fmt) {
            ///
            /// FASTA files
            ///
        case eFormat_FastA:
            {{
                CPluginHandle ph =
                    CPluginRegistry::GetPlugin("CDataPlugin_FileImport_Fasta");
                if (ph) {
                    CPluginCommand::ECommand cmd = CPluginCommand::eCommand_load;
                    if (project) {
                        cmd = CPluginCommand::eCommand_import;
                    }
                    msg = ph.CreateMessage(cmd);
                    disp_flags |= CPluginUtils::fResolveArgs;
                }
            }}
            break;

            ///
            /// NCBI serializable objects
            ///
        case eFormat_AsnBinary:
        case eFormat_AsnText:
        case eFormat_XML:
            {{
                CPluginHandle ph =
                    CPluginRegistry::GetPlugin("CDataPlugin_FileImport_Ncbi");
                if (ph) {
                    CPluginCommand::ECommand cmd = CPluginCommand::eCommand_load;
                    if (project) {
                        cmd = CPluginCommand::eCommand_import;
                    }
                    msg = ph.CreateMessage(cmd);
                    string serial_fmt;
                    if (fmt == eFormat_AsnBinary) {
                        serial_fmt = "ASN.1 Binary";
                    } else if (fmt == eFormat_AsnText) {
                        serial_fmt = "ASN.1 Text";
                    } else {
                        serial_fmt = "XML";
                    }
                    msg->SetRequest().SetCommand()["format"].SetString(serial_fmt);
                }
            }}
            break;

            ///
            /// AGP files
            ///
        case eFormat_AGP:
            {{
                CPluginHandle ph =
                    CPluginRegistry::GetPlugin("CDataPlugin_FileImport_Agp");
                if (ph) {
                    CPluginCommand::ECommand cmd = CPluginCommand::eCommand_load;
                    if (project) {
                        cmd = CPluginCommand::eCommand_import;
                    }
                    msg = ph.CreateMessage(cmd);
                    disp_flags |= CPluginUtils::fResolveArgs;
                }
            }}
            break;

            ///
            /// GTF files
            ///
        case eFormat_GTF:
            {{
                CPluginHandle ph =
                    CPluginRegistry::GetPlugin("CDataPlugin_FileImport_Gff");
                if (ph) {
                    CPluginCommand::ECommand cmd = CPluginCommand::eCommand_load;
                    if (project) {
                        cmd = CPluginCommand::eCommand_import;
                    }
                    msg = ph.CreateMessage(cmd);
                    disp_flags |= CPluginUtils::fResolveArgs;
                }
            }}
            break;

            ///
            /// Glimmer3 files
            ///
        case eFormat_Glimmer3:
            {{
                CPluginHandle ph =
                    CPluginRegistry::GetPlugin("CDataPlugin_FileImport_Glimmer");
                if (ph) {
                    CPluginCommand::ECommand cmd = CPluginCommand::eCommand_load;
                    if (project) {
                        cmd = CPluginCommand::eCommand_import;
                    }
                    msg = ph.CreateMessage(cmd);
                    disp_flags |= CPluginUtils::fResolveArgs;
                }
            }}
            break;

            ///
            /// Newick format tree files
            ///
        case eFormat_NewickTree:
            {{
                CPluginHandle ph =
                    CPluginRegistry::GetPlugin("CDataPlugin_FileImport_Newick");
                if (ph) {
                    CPluginCommand::ECommand cmd = CPluginCommand::eCommand_load;
                    if (project) {
                        cmd = CPluginCommand::eCommand_import;
                    }
                    msg = ph.CreateMessage(cmd);
                    disp_flags |= CPluginUtils::fResolveArgs;
                }
            }}
            break;

            ///
            /// Newick format tree files
            ///
        case eFormat_Phrap:
            {{
                CPluginHandle ph =
                    CPluginRegistry::GetPlugin("CDataPlugin_FileImport_Phrap");
                if (ph) {
                    CPluginCommand::ECommand cmd = CPluginCommand::eCommand_load;
                    if (project) {
                        cmd = CPluginCommand::eCommand_import;
                    }
                    msg = ph.CreateMessage(cmd);
                    disp_flags |= CPluginUtils::fResolveArgs;
                }
            }}
            break;

            ///
            /// Alignment files
            ///
        case eFormat_TextAlign:
            {{
                CPluginHandle ph =
                    CPluginRegistry::GetPlugin("CDataPlugin_FileImport_Alignment");
                if (ph) {
                    CPluginCommand::ECommand cmd = CPluginCommand::eCommand_load;
                    if (project) {
                        cmd = CPluginCommand::eCommand_import;
                    }
                    msg = ph.CreateMessage(cmd);
                    disp_flags |= CPluginUtils::fResolveArgs;
                }
            }}
            break;

            ///
            /// Repeatmasker out files
            ///
        case eFormat_RepeatMasker:
            {{
                CPluginHandle ph =
                    CPluginRegistry::GetPlugin( "CDataPlugin_FileImport_RepeatMasker" );
                if ( ph ) {
                    CPluginCommand::ECommand cmd = CPluginCommand::eCommand_load;
                    if ( project ) {
                        cmd = CPluginCommand::eCommand_import;
                    }
                    msg = ph.CreateMessage( cmd );
                    disp_flags |= CPluginUtils::fResolveArgs;
                }
            }}
            break;

            ///
            /// Distance Matrices
            ///
        case eFormat_DistanceMatrix:
            {{
                CPluginHandle ph =
                    CPluginRegistry::GetPlugin("CDataPlugin_FileImport_DistMat");
                if (ph) {
                    CPluginCommand::ECommand cmd = CPluginCommand::eCommand_load;
                    if (project) {
                        cmd = CPluginCommand::eCommand_import;
                    }
                    msg = ph.CreateMessage(cmd);

                    // NB: no options on import yet...
                    //disp_flags |= CPluginUtils::fResolveArgs;
                }
            }}
            break;

            ///
            /// Five-Column Feature Tables
            ///
        case eFormat_FiveColumn:
            {{
                CPluginHandle ph =
                    CPluginRegistry::GetPlugin("CDataPlugin_FileImport_FiveColumn");
                if (ph) {
                    CPluginCommand::ECommand cmd = CPluginCommand::eCommand_load;
                    if (project) {
                        cmd = CPluginCommand::eCommand_import;
                    }
                    msg = ph.CreateMessage(cmd);
                    disp_flags |= CPluginUtils::fResolveArgs;
                }
            }}
            break;

            ///
            /// Table files
            ///
        case eFormat_Table:
            {{
                CPluginHandle ph =
                    CPluginRegistry::GetPlugin("CDataPlugin_TableReader");
                if (ph) {
                    CPluginCommand::ECommand cmd = CPluginCommand::eCommand_load;
                    if (project) {
                        cmd = CPluginCommand::eCommand_import;
                    }
                    msg = ph.CreateMessage(cmd);

                    // NB: no options on import yet...
                    //disp_flags |= CPluginUtils::fResolveArgs;
                }
            }}
            break;

            ///
            /// Taxplot files
            ///
        case eFormat_Taxplot:
            {{
                CPluginHandle ph =
                    CPluginRegistry::GetPlugin("CDataPlugin_ShiReader");
                if (ph) {
                    CPluginCommand::ECommand cmd = CPluginCommand::eCommand_load;
                    if (project) {
                        cmd = CPluginCommand::eCommand_import;
                    }
                    msg = ph.CreateMessage(cmd);
                    disp_flags |= CPluginUtils::fResolveArgs;
                }
            }}
            break;

            ///
            /// GeneMark files
            ///
        case eFormat_GeneMark:
            {{
                CPluginHandle ph =
                    CPluginRegistry::GetPlugin("CDataPlugin_GeneMark");
                if (ph) {
                    CPluginCommand::ECommand cmd = CPluginCommand::eCommand_load;
                    if (project) {
                        cmd = CPluginCommand::eCommand_import;
                    }
                    msg = ph.CreateMessage(cmd);
                    disp_flags |= CPluginUtils::fResolveArgs;
                }
            }}
            break;

            ///
            /// SNP Markers file
            ///
        case eFormat_SNPMarkers:
            {{
                CPluginHandle ph =
                    CPluginRegistry::GetPlugin("CDataPlugin_FileImport_Smk");
                if (ph) {
                    CPluginCommand::ECommand cmd = CPluginCommand::eCommand_load;
                    if (project) {
                        cmd = CPluginCommand::eCommand_import;
                    }
                    msg = ph.CreateMessage(cmd);
                    // NB: no options on import yet...
                    disp_flags |= CPluginUtils::fResolveArgs;
                }
            }}
            break;

        default:
        case eFormat_Unknown:
            {{
                string str("Cannot guess file type from\n");
                str += fname;
                reply.AddAction(CPluginReplyAction::e_Error, str);
            }}
            break;
        }

        if (msg) {
            msg->SetRequest().SetCommand()["file"].SetInputFile(fname);
            if (project) {
                msg->SetRequest().SetCommand()["project"].SetProject(*project);
            }
            CPluginUtils::CallPlugin(*msg,
                                     CPluginUtils::eDispatch_Deferred,
                                     disp_flags);
            reply.SetStatus(eMessageStatus_success);
        } else {
            string msg = "Can't interpret file " + fname;
            reply.AddAction(CPluginReplyAction::e_Error, msg);
        }
    } else {
        // file does not exists - report error
        string msg = string("File \"") + fname + string("\" does not exist,");
        reply.AddAction(CPluginReplyAction::e_Error, msg);
    }
}


CDataPlugin_FileImport::EFormat
CDataPlugin_FileImport::x_GetFormat(CNcbiIfstream& istr)
{
    //
    //  We want to make a reasonable effort of detecting alignment files.
    //  For dependency reasons, the following test cannot go into CFormatGuess.
    //  Also, we need to make this test before CFormatGuess checks for fastA
    //  because some alignment files also pass as perfectly fine fastA files.
    //
    //  Note: For big files, we still have the problem that alignment files
    //  may be recognized as fastA instead. I just don't know what to do about
    //  that (ludwigf, 2006-08-17).
    //
    CT_POS_TYPE orig_pos = istr.tellg();

    CFormatGuess fg( istr );
    CFormatGuess::EFormat fmt = fg.GuessFormat();
    switch (fmt) {
    case CFormatGuess::eBinaryASN:
        return eFormat_AsnBinary;
    case CFormatGuess::eTextASN:
        return eFormat_AsnText;
    case CFormatGuess::eXml:
        return eFormat_XML;
    case CFormatGuess::eFasta:
        return eFormat_FastA;
    case CFormatGuess::eRmo:
        return eFormat_RepeatMasker;
    case CFormatGuess::ePhrapAce:
        return eFormat_Phrap;
    case CFormatGuess::eAgp:
        return eFormat_AGP;
    case CFormatGuess::eGtf:
        return eFormat_GTF;
    case CFormatGuess::eGlimmer3:
        return eFormat_Glimmer3;
    case CFormatGuess::eNewick:
        return eFormat_NewickTree;
    case CFormatGuess::eDistanceMatrix:
        return eFormat_DistanceMatrix;
    case CFormatGuess::eFiveColFeatureTable:
        return eFormat_FiveColumn;
    case CFormatGuess::eTaxplot:
        return eFormat_Taxplot;
    case CFormatGuess::eTable:
        return eFormat_Table;
#if 0
    case CFormatGuess::eSnpMarkers:
        return eFormat_SNPMarkers;
#endif

    default:
        break;
    }

    return eFormat_Unknown;
}


END_NCBI_SCOPE

---