00001
00002
00003
00004
00005
00006
00007
00008
00009
00010
00011
00012
00013
00014
00015
00016
00017
00018
00019
00020
00021
00022
00023
00024
00025
00026
00027
00028
00029
00030
00031
00032 #include <ncbi_pch.hpp>
00033 #include <corelib/ncbistd.hpp>
00034 #include <corelib/ncbiargs.hpp>
00035
00036 #include <serial/objistr.hpp>
00037 #include <serial/objostr.hpp>
00038 #include <serial/objcopy.hpp>
00039
00040 #include <memory>
00041
00042 #include "exceptions.hpp"
00043 #include "code.hpp"
00044 #include "lexer.hpp"
00045 #include "dtdlexer.hpp"
00046 #include "parser.hpp"
00047 #include "dtdparser.hpp"
00048 #include "xsdlexer.hpp"
00049 #include "xsdparser.hpp"
00050 #include "moduleset.hpp"
00051 #include "module.hpp"
00052 #include "type.hpp"
00053 #include "generate.hpp"
00054 #include "datatool.hpp"
00055 #include "filecode.hpp"
00056 #include <serial/objistrxml.hpp>
00057 #include <serial/objostrxml.hpp>
00058 #include <serial/error_codes.hpp>
00059
00060 #include <common/test_assert.h>
00061
00062
00063 #define NCBI_USE_ERRCODE_X Serial_DataTool
00064
00065
00066 BEGIN_NCBI_SCOPE
00067
00068 int CDataTool::Run(void)
00069 {
00070 if ( !ProcessModules() )
00071 return 1;
00072 if ( !ProcessData() )
00073 return 1;
00074 if ( !GenerateCode() )
00075 return 1;
00076
00077 return 0;
00078 }
00079
00080 CDataTool::CDataTool(void)
00081 {
00082 SetVersion( CVersionInfo(1,9,2) );
00083 }
00084
00085 void CDataTool::Init(void)
00086 {
00087 SetDiagPostLevel(eDiag_Warning);
00088
00089 auto_ptr<CArgDescriptions> d(new CArgDescriptions);
00090
00091 d->SetUsageContext("datatool", "work with ASN.1/XML data");
00092
00093
00094 d->AddKey("m", "moduleFile",
00095 "module file(s)",
00096 CArgDescriptions::eString,
00097 CArgDescriptions::fAllowMultiple);
00098 d->AddDefaultKey("M", "externalModuleFile",
00099 "external module file(s)",
00100 CArgDescriptions::eString, NcbiEmptyString,
00101 CArgDescriptions::fAllowMultiple);
00102 d->AddFlag("i",
00103 "ignore unresolved symbols");
00104 d->AddOptionalKey("f", "moduleFile",
00105 "write ASN.1 module file",
00106 CArgDescriptions::eOutputFile);
00107 d->AddOptionalKey("fx", "dtdFile",
00108 "write DTD file (\"-fx m\" writes modular DTD file)",
00109 CArgDescriptions::eOutputFile);
00110 d->AddOptionalKey("fxs", "XMLSchemaFile",
00111 "write XML Schema file (\"-fxs m\" writes modular Schema file)",
00112 CArgDescriptions::eOutputFile);
00113 d->AddOptionalKey("fd", "SpecificationDump",
00114 "write specification dump file (datatool internal format)",
00115 CArgDescriptions::eOutputFile);
00116 d->AddOptionalKey("ms", "moduleSuffix",
00117 "suffix of modular DTD or Schema file name",
00118 CArgDescriptions::eString);
00119
00120
00121 d->AddOptionalKey("v", "valueFile",
00122 "read value in ASN.1 text format",
00123 CArgDescriptions::eInputFile);
00124 d->AddOptionalKey("vx", "valueFile",
00125 "read value in XML format",
00126 CArgDescriptions::eInputFile);
00127 d->AddOptionalKey("d", "valueFile",
00128 "read value in ASN.1 binary format (-t is required)",
00129 CArgDescriptions::eInputFile);
00130 d->AddOptionalKey("t", "type",
00131 "binary value type (see \"-d\" argument)",
00132 CArgDescriptions::eString);
00133 d->AddOptionalKey("dn", "filename",
00134 "DTD module name in XML header (no extension). "
00135 "If empty, omit DOCTYPE line.",
00136 CArgDescriptions::eString);
00137 d->AddFlag("F",
00138 "read value completely into memory");
00139 d->AddOptionalKey("p", "valueFile",
00140 "write value in ASN.1 text format",
00141 CArgDescriptions::eOutputFile);
00142 d->AddOptionalKey("px", "valueFile",
00143 "write value in XML format",
00144 CArgDescriptions::eOutputFile);
00145 d->AddOptionalKey("pj", "valueFile",
00146 "write value in JSON format",
00147 CArgDescriptions::eOutputFile);
00148 d->AddOptionalKey("xmlns", "namespaceName",
00149 "XML namespace name",
00150 CArgDescriptions::eString);
00151 d->AddOptionalKey("e", "valueFile",
00152 "write value in ASN.1 binary format",
00153 CArgDescriptions::eOutputFile);
00154 d->AddFlag("sxo",
00155 "no scope prefixes in XML output");
00156 d->AddFlag("sxi",
00157 "no scope prefixes in XML input");
00158
00159
00160 d->AddOptionalKey("oex", "exportSpec",
00161 "class export specifier for MSVC",
00162 CArgDescriptions::eString);
00163 d->AddOptionalKey("od", "defFile",
00164 "code definition file",
00165 CArgDescriptions::eInputFile);
00166 d->AddFlag("odi",
00167 "silently ignore absent code definition file");
00168 d->AddFlag("odw",
00169 "issue a warning about absent code definition file");
00170 d->AddOptionalKey("of", "listFile",
00171 "write list of generated C++ files",
00172 CArgDescriptions::eOutputFile);
00173 d->AddOptionalKey("oc", "basename",
00174 "write combining C++ files",
00175 CArgDescriptions::eString);
00176 d->AddFlag("oA",
00177 "generate C++ files for all types");
00178 d->AddOptionalKey("ot", "types",
00179 "generate C++ files for listed types",
00180 CArgDescriptions::eString);
00181 d->AddOptionalKey("ox", "types",
00182 "exclude listed types from generation",
00183 CArgDescriptions::eString);
00184 d->AddFlag("oX",
00185 "turn off recursive type generation");
00186
00187 d->AddOptionalKey("on", "namespace",
00188 "default namespace",
00189 CArgDescriptions::eString);
00190
00191 d->AddOptionalKey("opm", "directory",
00192 "directory for searching source modules",
00193 CArgDescriptions::eString);
00194 d->AddOptionalKey("oph", "directory",
00195 "directory for generated *.hpp files",
00196 CArgDescriptions::eString);
00197 d->AddOptionalKey("opc", "directory",
00198 "directory for generated *.cpp files",
00199 CArgDescriptions::eString);
00200
00201 d->AddOptionalKey("or", "prefix",
00202 "add prefix to generated file names",
00203 CArgDescriptions::eString);
00204 d->AddFlag("orq",
00205 "use quoted syntax form for generated include files");
00206 d->AddFlag("ors",
00207 "add source file dir to generated file names");
00208 d->AddFlag("orm",
00209 "add module name to generated file names");
00210 d->AddFlag("orA",
00211 "combine all -or* prefixes");
00212 d->AddFlag("ocvs",
00213 "create \".cvsignore\" files");
00214 d->AddOptionalKey("oR", "rootDirectory",
00215 "set \"-o*\" arguments for NCBI directory tree",
00216 CArgDescriptions::eString);
00217
00218 d->AddFlag("oDc",
00219 "turn on generation of DOXYGEN-style comments");
00220 d->AddOptionalKey("odx", "URL",
00221 "URL of documentation root folder (for DOXYGEN)",
00222 CArgDescriptions::eString);
00223 d->AddFlag("lax_syntax",
00224 "allow non-standard ASN.1 syntax accepted by asntool");
00225 d->AddOptionalKey("pch", "file",
00226 "name of the precompiled header to include in all *.cpp files",
00227 CArgDescriptions::eString);
00228
00229 SetupArgDescriptions(d.release());
00230 }
00231
00232 bool CDataTool::ProcessModules(void)
00233 {
00234 const CArgs& args = GetArgs();
00235
00236
00237 if ( const CArgValue& od = args["od"] )
00238 generator.LoadConfig(od.AsString(), args["odi"], args["odw"]);
00239
00240 list<string> modulesPath;
00241 string opt;
00242
00243 if ( generator.GetOpt("oR", &opt) ) {
00244
00245 const string& rootDir = opt;
00246 generator.SetRootDir(rootDir);
00247 generator.SetHPPDir(Path(rootDir, "include"));
00248 string srcDir = Path(rootDir, "src");
00249 generator.SetCPPDir(srcDir);
00250 modulesPath.push_back(srcDir);
00251 generator.SetFileNamePrefixSource(eFileName_FromSourceFileName);
00252 generator.SetDefaultNamespace("NCBI_NS_NCBI::objects");
00253 }
00254
00255 if ( generator.GetOpt("opm", &opt) ) {
00256
00257 NStr::Split(opt, ",", modulesPath);
00258 }
00259
00260 SourceFile::EType srctype =
00261 LoadDefinitions(generator.GetMainModules(),
00262 modulesPath, args["m"].GetStringList(), false);
00263
00264 if (srctype == SourceFile::eASN) {
00265 LoadDefinitions(generator.GetImportModules(),
00266 modulesPath, args["M"].GetStringList(), true, srctype);
00267 }
00268
00269 if ( args["sxo"] ) {
00270 CDataType::SetEnforcedStdXml(true);
00271 }
00272
00273 if ( const CArgValue& f = args["f"] ) {
00274 generator.GetMainModules().PrintASN(f.AsOutputFile());
00275 f.CloseFile();
00276 }
00277 if ( const CArgValue& f = args["fd"] ) {
00278 generator.GetMainModules().PrintSpecDump(f.AsOutputFile());
00279 f.CloseFile();
00280 }
00281
00282 if ( const CArgValue& fx = args["fx"] ) {
00283 if (srctype == SourceFile::eDTD || srctype == SourceFile::eXSD) {
00284 CDataType::SetEnforcedStdXml(true);
00285 }
00286 if ( fx.AsString() == "m" ) {
00287 if ( const CArgValue& ms = args["ms"] ) {
00288 CDataTypeModule::SetModuleFileSuffix(ms.AsString());
00289 }
00290 generator.ResolveImportRefs();
00291 CDataType::EnableDTDEntities(true);
00292 generator.GetMainModules().PrintDTDModular();
00293 } else {
00294 generator.GetMainModules().PrintDTD(fx.AsOutputFile());
00295 fx.CloseFile();
00296 }
00297 }
00298
00299 if ( const CArgValue& ax = args["fxs"] ) {
00300 if (srctype == SourceFile::eDTD || srctype == SourceFile::eXSD) {
00301 CDataType::SetEnforcedStdXml(true);
00302 }
00303 if ( ax.AsString() == "m" ) {
00304 if ( const CArgValue& ms = args["ms"] ) {
00305 CDataTypeModule::SetModuleFileSuffix(ms.AsString());
00306 }
00307 generator.ResolveImportRefs();
00308 generator.GetMainModules().PrintXMLSchemaModular();
00309 } else {
00310 generator.GetMainModules().PrintXMLSchema(ax.AsOutputFile());
00311 ax.CloseFile();
00312 }
00313 }
00314
00315 if ( !generator.Check() ) {
00316 if ( !args["i"] ) {
00317 ERR_POST_X(1, "some types are unknown");
00318 return false;
00319 }
00320 else {
00321 ERR_POST_X(2, Warning << "some types are unknown: ignoring");
00322 }
00323 }
00324 return true;
00325 }
00326
00327 bool CDataTool::ProcessData(void)
00328 {
00329 const CArgs& args = GetArgs();
00330 bool stdXmlIn = false;
00331 if ( args["sxi"] ) {
00332 stdXmlIn = true;
00333 }
00334 bool stdXmlOut = false;
00335 if ( args["sxo"] ) {
00336 stdXmlOut = true;
00337 }
00338
00339
00340 ESerialDataFormat inFormat;
00341 string inFileName;
00342 const CArgValue& t = args["t"];
00343
00344 if ( const CArgValue& v = args["v"] ) {
00345 inFormat = eSerial_AsnText;
00346 inFileName = v.AsString();
00347 }
00348 else if ( const CArgValue& vx = args["vx"] ) {
00349 inFormat = eSerial_Xml;
00350 inFileName = vx.AsString();
00351 }
00352 else if ( const CArgValue& d = args["d"] ) {
00353 if ( !t ) {
00354 ERR_POST_X(3, "ASN.1 value type must be specified (-t)");
00355 return false;
00356 }
00357 inFormat = eSerial_AsnBinary;
00358 inFileName = d.AsString();
00359 }
00360 else
00361 return true;
00362
00363 auto_ptr<CObjectIStream>
00364 in(CObjectIStream::Open(inFormat, inFileName, eSerial_StdWhenAny));
00365 if (inFormat == eSerial_Xml) {
00366 CObjectIStreamXml *is = dynamic_cast<CObjectIStreamXml*>(in.get());
00367 if (stdXmlIn) {
00368 is->SetEnforcedStdXml(true);
00369 }
00370 is->SetDefaultStringEncoding(eEncoding_Unknown);
00371 }
00372
00373 string typeName;
00374 if ( t ) {
00375 typeName = t.AsString();
00376 in->ReadFileHeader();
00377 }
00378 else {
00379 typeName = in->ReadFileHeader();
00380 }
00381
00382 TTypeInfo typeInfo =
00383 generator.GetMainModules().ResolveInAnyModule(typeName, true)->
00384 GetTypeInfo().Get();
00385
00386
00387 ESerialDataFormat outFormat;
00388 string outFileName;
00389 bool use_nsName = false;
00390 string nsName;
00391
00392 if ( const CArgValue& p = args["p"] ) {
00393 outFormat = eSerial_AsnText;
00394 outFileName = p.AsString();
00395 }
00396 else if ( const CArgValue& px = args["px"] ) {
00397 outFormat = eSerial_Xml;
00398 outFileName = px.AsString();
00399 if ( const CArgValue& px_ns = args["xmlns"] ) {
00400 use_nsName = true;
00401 nsName = px_ns.AsString();
00402 }
00403 }
00404 else if ( const CArgValue& pj = args["pj"] ) {
00405 outFormat = eSerial_Json;
00406 outFileName = pj.AsString();
00407 }
00408 else if ( const CArgValue& e = args["e"] ) {
00409 outFormat = eSerial_AsnBinary;
00410 outFileName = e.AsString();
00411 }
00412 else {
00413
00414 outFormat = eSerial_None;
00415 }
00416
00417 if ( args["F"] ) {
00418
00419 AnyType value;
00420 in->Read(&value, typeInfo, CObjectIStream::eNoFileHeader);
00421 if ( outFormat != eSerial_None ) {
00422
00423 auto_ptr<CObjectOStream>
00424 out(CObjectOStream::Open(outFormat, outFileName,
00425 eSerial_StdWhenAny));
00426 if ( outFormat == eSerial_Xml ) {
00427 CObjectOStreamXml *os = dynamic_cast<CObjectOStreamXml*>(out.get());
00428 if (stdXmlOut) {
00429 os->SetEnforcedStdXml(true);
00430 }
00431 os->SetDefaultStringEncoding(eEncoding_Unknown);
00432 if (use_nsName) {
00433 os->SetReferenceSchema(true);
00434 if (!nsName.empty()) {
00435 os->SetDefaultSchemaNamespace(nsName);
00436 }
00437 }
00438
00439 if( const CArgValue& dn = args["dn"] ) {
00440 const string& name = dn.AsString();
00441 if ( name.empty() ) {
00442 os->SetReferenceDTD(false);
00443 }
00444 else {
00445 os->SetReferenceDTD(true);
00446 os->SetDTDFileName(name);
00447 }
00448 }
00449 }
00450 out->Write(&value, typeInfo);
00451 }
00452 }
00453 else {
00454 if ( outFormat != eSerial_None ) {
00455
00456 auto_ptr<CObjectOStream>
00457 out(CObjectOStream::Open(outFormat, outFileName,
00458 eSerial_StdWhenAny));
00459 if ( outFormat == eSerial_Xml ) {
00460 CObjectOStreamXml *os = dynamic_cast<CObjectOStreamXml*>(out.get());
00461 if (stdXmlOut) {
00462 os->SetEnforcedStdXml(true);
00463 }
00464 os->SetDefaultStringEncoding(eEncoding_Unknown);
00465 if (use_nsName) {
00466 os->SetReferenceSchema(true);
00467 if (!nsName.empty()) {
00468 os->SetDefaultSchemaNamespace(nsName);
00469 }
00470 }
00471
00472 if( const CArgValue& dn = args["dn"] ) {
00473 const string& name = dn.AsString();
00474 if ( name.empty() ) {
00475 os->SetReferenceDTD(false);
00476 }
00477 else {
00478 os->SetReferenceDTD(true);
00479 os->SetDTDFileName(name);
00480 }
00481 }
00482 }
00483 CObjectStreamCopier copier(*in, *out);
00484 copier.Copy(typeInfo, CObjectStreamCopier::eNoFileHeader);
00485
00486
00487 for (bool go=true; go; ) {
00488 try {
00489 copier.Copy(typeInfo);
00490 } catch (CEofException&) {
00491 go = false;
00492 }
00493 }
00494 }
00495 else {
00496
00497 in->Skip(typeInfo, CObjectIStream::eNoFileHeader);
00498 }
00499 }
00500 return true;
00501 }
00502
00503 bool CDataTool::GenerateCode(void)
00504 {
00505 string opt;
00506
00507
00508
00509
00510 if ( generator.GetOpt("oX") )
00511 generator.ExcludeRecursion();
00512 if ( generator.GetOpt("oA") )
00513 generator.IncludeAllMainTypes();
00514 if ( generator.GetOpt("ot", &opt) )
00515 generator.IncludeTypes(opt);
00516 if ( generator.GetOpt("ox", &opt) )
00517 generator.ExcludeTypes(opt);
00518
00519 if ( !generator.HaveGenerateTypes() )
00520 return true;
00521
00522
00523 if ( generator.GetOpt("oex", &opt) ) {
00524 string ex;
00525 ex = generator.GetConfig().Get("-","_export");
00526 if (ex.empty()) {
00527 ex = opt;
00528 }
00529 CClassCode::SetExportSpecifier(ex);
00530 }
00531
00532 {
00533 if ( generator.GetOpt("oDc") ) {
00534 CClassCode::SetDoxygenComments(true);
00535 if ( generator.GetOpt("odx", &opt) ) {
00536 string root = opt;
00537 if (root.empty()) {
00538
00539 root = "http://www.ncbi.nlm.nih.gov/IEB/ToolBox/CPP_DOC/lxr/source";
00540 }
00541 CClassCode::SetDocRootURL(root);
00542 }
00543 string group = generator.GetConfig().Get("-","_addtogroup_name");
00544 CClassCode::SetDoxygenGroup(group);
00545 group = generator.GetConfig().Get("-","_ingroup_name");
00546 generator.SetDoxygenIngroup(group);
00547 group = generator.GetConfig().Get("-","_addtogroup_description");
00548 generator.SetDoxygenGroupDescription(group);
00549 } else {
00550 CClassCode::SetDoxygenComments(false);
00551 }
00552 }
00553
00554
00555
00556
00557 opt = "";
00558 if ( generator.GetOpt("on", &opt) ) {
00559 generator.SetDefaultNamespace(opt);
00560 } else if (opt == "-") {
00561 generator.ResetDefaultNamespace();
00562 }
00563
00564
00565 if ( generator.GetOpt("oc", &opt) ) {
00566 const string& fileName = opt;
00567 generator.SetCombiningFileName(fileName);
00568 generator.SetFileListFileName(fileName+".files");
00569 }
00570 if ( generator.GetOpt("of", &opt) )
00571 generator.SetFileListFileName(opt);
00572
00573
00574 if ( generator.GetOpt("oph", &opt) )
00575 generator.SetHPPDir(opt);
00576 if ( generator.GetOpt("opc", &opt) )
00577 generator.SetCPPDir(opt);
00578
00579
00580 if ( generator.GetOpt("or", &opt) )
00581 generator.SetFileNamePrefix(opt);
00582 if ( generator.GetOpt("orq") )
00583 generator.UseQuotedForm(true);
00584 if ( generator.GetOpt("ocvs") )
00585 generator.CreateCvsignore(true);
00586 if ( generator.GetOpt("ors") )
00587 generator.SetFileNamePrefixSource(eFileName_FromSourceFileName);
00588 if ( generator.GetOpt("orm") )
00589 generator.SetFileNamePrefixSource(eFileName_FromModuleName);
00590 if ( generator.GetOpt("orA") )
00591 generator.SetFileNamePrefixSource(eFileName_UseAllPrefixes);
00592
00593
00594 if ( generator.GetOpt("pch", &opt) )
00595 CFileCode::SetPchHeader(opt);
00596
00597
00598 generator.GenerateCode();
00599 return true;
00600 }
00601
00602
00603 SourceFile::EType CDataTool::LoadDefinitions(
00604 CFileSet& fileSet, const list<string>& modulesPath,
00605 const CArgValue::TStringArray& nameList,
00606 bool split_names, SourceFile::EType srctype)
00607 {
00608 SourceFile::EType moduleType;
00609 list<string> names;
00610
00611 ITERATE (CArgValue::TStringArray, n, nameList) {
00612 if (split_names) {
00613 list<string> t;
00614 NStr::Split(*n, " ", t);
00615 names.insert(names.end(), t.begin(), t.end());
00616 } else {
00617 names.push_back(*n);
00618 }
00619 }
00620
00621 ITERATE ( list<string>, fi, names ) {
00622 const string& name = *fi;
00623 if ( !name.empty() ) {
00624 SourceFile fName(name, modulesPath);
00625 moduleType = fName.GetType();
00626
00627
00628 if (srctype == SourceFile::eUnknown) {
00629 if (moduleType == SourceFile::eUnknown) {
00630 moduleType = SourceFile::eASN;
00631 }
00632 srctype = moduleType;
00633 }
00634
00635 else {
00636 if (moduleType == SourceFile::eUnknown) {
00637 moduleType = srctype;
00638 }
00639
00640 else if (moduleType != srctype) {
00641 NCBI_THROW(CDatatoolException,eWrongInput,
00642 "Unable to process modules of different types"
00643 " simultaneously: "+name);
00644 }
00645 }
00646
00647 switch (moduleType) {
00648 default:
00649 NCBI_THROW(CDatatoolException,eWrongInput,"Unknown file type: "+name);
00650
00651 case SourceFile::eASN:
00652 {
00653 ASNLexer lexer(fName,name);
00654 lexer.AllowIDsEndingWithMinus(generator.GetOpt("lax_syntax"));
00655 ASNParser parser(lexer);
00656 fileSet.AddFile(parser.Modules(name));
00657 }
00658 break;
00659 case SourceFile::eDTD:
00660 {
00661 DTDLexer lexer(fName,name);
00662 DTDParser parser(lexer);
00663 fileSet.AddFile(parser.Modules(name));
00664 CDataType::SetXmlSourceSpec(true);
00665 }
00666 break;
00667 case SourceFile::eXSD:
00668 {
00669 XSDLexer lexer(fName,name);
00670 XSDParser parser(lexer);
00671 fileSet.AddFile(parser.Modules(name));
00672 CDataType::SetXmlSourceSpec(true);
00673 }
00674 break;
00675 }
00676 }
00677 }
00678 return srctype;
00679 }
00680
00681 END_NCBI_SCOPE
00682
00683
00684 int main(int argc, const char* argv[])
00685 {
00686 USING_NCBI_SCOPE;
00687 CException::EnableBackgroundReporting(false);
00688 return CDataTool().AppMain(argc, argv, 0, eDS_Default, 0, "datatool");
00689 }
00690
00691