Something went wrong. Try again.
This repository has no description
Something went wrong. Try again.
60 kB · 991 lines
Python
at develop
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992#!/usr/bin/env python3
## Author: Gleb Belov@monash.edu 2017## This program runs MiniZinc solvers over a set of instances,## checks solutions and compares to given solution logs.
## TODO type errors etc. when checking?## TODO -a for optimization only## TODO continuous output dumping as option## TODO CPU/user time limit, proper memory limit (setrlimit not working)
import sys, io, re as regex, tracebackimport os.path, platform##import numpyimport mathimport json, argparseimport datetimefrom collections import OrderedDict
import utils, json_config, json_log, mzn_exec, cmp_result_logsfrom json_config import s_CommentKey, s_AddKey
s_ProgramDescr = 'MiniZinc testing automation. (c) 2018 Monash University, gleb.belov@monash.edu's_ProgramDescrLong = ( "Allows checking of MiniZinc solutions by configurable checker profiles. The solutions can be input from previous logs or produced by a chosen solver profile, for 1 or several instances, with result comparison and some ranking. New solutions' summary logs, detailed stdout/err outputs, and statistics are saved in subfolder mzn-test/LOGS, /OUTPUTS, and /STATS, resp.")
########################### GENERAL CONFIG. Could be in the JSON config actually #########################sDirResults = "mzn-test"sFlnSolUnchk = sDirResults + "/sol__unchk.json" ### Logfile to append immediate resultssFlnSolCheckBase = sDirResults + "/LOGS/sol__{}.json" ### Logfile to append checked results. {} replaced by datetimesFlnSolFailBase = sDirResults + "/LOGS/sol__{}__FAIL.json" ### To save failed solutionssFlnSolLastDzn = sDirResults + "/sol_last.dzn" ### File to save the DZN solution for checkingsFlnStdoutBase = sDirResults + "/OUTPUTS/{}.stdout.txt" ### File name base to dump stdout for any backend callsFlnStderrBase = sDirResults + "/OUTPUTS/{}.stderr.txt"sFlnStatLog = sDirResults + "/STATS/stat_log__{}.txt" ### The instance list name(s) will be inserted, if any
sDZNOutputAgrs = "--output-mode dzn --output-objective" ## The flattener arguments to produce DZN-compatible output facilitating solution checking ## Remove --output-objective in MZN Release <=2.1.7sFlatOptChecker = "--allow-multiple-assignments"
s_UsageExamples = ( "\nUSAGE EXAMPLES:" "\n(1) \"mzn-test.py model.mzn data.dzn [--checkDZN stdout.txt [--checkStderr stderr.txt]] [--chkPrf MINIZINC-CHK --chkPrf FZN-GECODE-CHK] [--tCheck 15] [--addSolverOption \"--fzn-flags '-D fPureCircuit=true'\"]\" ::: check the instance's solutions, optionally reading them from a DZN-formatted file (otherwise solving first), optionally overriding default checker list etc." "\n(2) \"mzn-test.py --solver CPLEX -t 300 -l instList1.txt -l instList2.txt --name CPLEXTest_003 --result newLog00.json prevLog1.json prevLog2.json --failed failLog.json\"" " ::: solve instances using the specified solver profile and wall time limit 300 seconds. The instances are taken from the list files. The test is aliased CPLEXTest_003. Results are saved to newLog00.json and compared/ranked to those in prevLog's. (Probably) incorrect solutions are saved to failLog.json." "\n(3) \"mzn-test.py [-l instList1.txt] -c prevLog1.json -c prevLog2.json [--runAndCmp]\" ::: compare existing logs, optionally limited to the given instances, optionally running new tests. USE SINGLE QUOTES ONLY INSIDE ARGUMENTS PASSED TO THE BACKENDS when running backends through shell." )############################################################################################################## Parameters of MZN-Test, including config and command-line##############################################################################################class MZT_Param: def parseCmdLine(self): parser = argparse.ArgumentParser( description=s_ProgramDescr + '\n' + s_ProgramDescrLong, epilog=s_UsageExamples) parser.add_argument('instanceFiles', nargs='*', metavar='<instanceFile>', help='model instance files, if no instance lists supplied, otherwise existing solution logs to compare with') parser.add_argument('-l', '--instanceList', dest='l_InstLists', action='append', metavar='<instanceListFile>', help='file with a list of instance input files, one instance per line,' ' instance file types specified in config. Used for running tests or for instance selection in comparison mode') parser.add_argument('--name', '--testName', metavar='<string>', help='alias of this test run, defaults to result log file name') parser.add_argument('-c', '--compare', action="append", metavar='<logfile>', help='summarize and compare results to existing <logfile>. Only compares the logs and does not run tests, unless --runAndCmp. The flags -c can be omitted if -l is used') parser.add_argument('--runAndCmp', '--runAndCompare', '--run', action='store_true', help='even if other logs are provided by -c, do run the tests and compare') parser.add_argument('--solver', '--solverPrf', '--solverProfile', metavar='<prf_name>', help='solver profile from those defined in config section \"SOLVER_PROFILES\"') parser.add_argument('--call', '--solverCall', metavar='"<exe+flags or shell command(s) if --shellSolve 1>"', help='solver backend call, should be quoted. Insert %%s where instance files need to be. Flatten with \'' + sDZNOutputAgrs + '\' to enable solution checking, unless the model has a suitable output definition.' " Pass '--output-time' to the output filter (e.g., solns2out) to enable ranking by time") parser.add_argument('-t', '--tSolve', type=float, metavar='<sec>', help='solver backend wall-time limit, default: '+ str(self.cfgDefault["BACKEND_DEFS"]["__BE_SOLVER"]["EXE"]["n_TimeoutRealHard"][0])) parser.add_argument('--result', default=sFlnSolCheckBase, metavar='<file>', help='save result log to <file>, default: \''+sFlnSolCheckBase+'\'') parser.add_argument('--resultUnchk', default=sFlnSolUnchk, metavar='<file>', help='save unchecked result log to <file>') parser.add_argument('--chkPrf', '--checkerPrf', '--checkerProfile', metavar='<prf_name>', action='append', help='checker profile from those defined in config section \"CHECKER_PROFILES\", can be a few') parser.add_argument('--tCheck', type=float, metavar='<sec>', help='checker backend wall-time limit, default: '+ str(self.cfgDefault["BACKEND_DEFS"]["__BE_CHECKER"]["EXE"]["n_TimeoutRealHard"][0])) parser.add_argument('--nCheckMax', '--nCheck', '--nCheckedMax', ## default=self.cfgDefault["COMMON_OPTIONS"]["SOLUTION_CHECKING"]["s_FailedSaveFile"][0], type=int, metavar='<N>', help='max number of solutions checked per instance.' ' Negative means checking starts from the last obtained solution. Default: '+ str(self.cfgDefault["COMMON_OPTIONS"]["SOLUTION_CHECKING"]["n_CheckedMax"][0])) parser.add_argument('--nFailedSaveMax', ## default=self.cfgDefault["COMMON_OPTIONS"]["SOLUTION_CHECKING"]["s_FailedSaveFile"][0], type=int, metavar='<N>', help='max number of failed solution reports saved per instance, default: '+ str(self.cfgDefault["COMMON_OPTIONS"]["SOLUTION_CHECKING"]["n_FailedSaveMax"][0])) parser.add_argument('--failed', ## default=self.cfgDefault["COMMON_OPTIONS"]["SOLUTION_CHECKING"]["s_FailedSaveFile"][0], metavar='<file>', help='save failed check reports to <file>, default: \''+ self.cfgDefault["COMMON_OPTIONS"]["SOLUTION_CHECKING"]["s_FailedSaveFile"][0]+'\'') parser.add_argument('--checkDZN', '--checkStdout', metavar='<stdout_file>', help='for a single instance, check DZN-formatted solutions from a solver\'s std output dumped to <stdout_file>. The DZN format is produced, e.g., if the model is flattened with \'' + sDZNOutputAgrs + '\'') parser.add_argument('--checkStderr', metavar='<stderr_file>', help='with --checkDZN, read a solver\'s stderr log from <stderr_file> (not essential)') parser.add_argument('-v', '--verbose', dest='verbose', action='store_true', help='tee backend\'s stderr to screen, in addition to the instance\'s output dumpfile in mzn-test/OUTPUTS. Only works for --shellSolve 1') parser.add_argument('--vc', '--verbose-check', dest='vc', action='store_true', help='same for checking') ## parser.add_argument('--no-feature', dest='feature', action='store_false') ## parser.set_defaults(feature=True) parser.add_argument('--debug', '--printcall', type=int, metavar='<bitfield>', help='bit 1: print full solver call commands, bit 2: same for checker') parser.add_argument('--shellSolve', type=int, metavar='0/1', help='backend call through shell when using psutils') parser.add_argument('--psutils', type=int, metavar='0/1', help='backend call through psutils (seems buggy in 3.4.2)') ## parser.add_argument('--fullPaths', action='store_true', ## help='use full paths in instance identifiers. By default, it\'s the pure base filenames') parser.add_argument('--mergeCfg', action="append", metavar='<file>', help='merge config from <file>') parser.add_argument('--saveCfg', metavar='<file>', help='save internal config to <file>. Can be useful to modify some parameters and run with --mergeCfg') parser.add_argument('--saveSolverCfg', metavar='<file>', help='save the final solver backend config to <file>') parser.add_argument('--saveCheckerCfg', metavar='<file>', help='save the final checker backend config to <file>') parser.add_argument('--addOption', '--addOptions', action="append", metavar='<text>', type=str, help='add <text> to any solver / checker call') parser.add_argument('--addSolverOption', '--addSolverOptions', action="append", metavar='<text>', type=str, help='add <text> to the solver call') parser.add_argument('--addCheckerOption', '--addCheckerOptions', action="append", metavar='<text>', type=str, help='add <text> to a checker call') parser.add_argument('--useJoinedName', action="append", metavar='<...%s...>', type=str, help='add this to the call, with %%s being replaced by' ' a joined filename from all the input filenames, e.g., "--writeModel MODELS/%%s.mps"') self.args = parser.parse_args() # print( "ARGS:\n", self.args ) ## Solver backend and checker backend list self.slvBE = None self.chkBEs = None
## Get parameters from config and command line, merge values def obtainParams(self): self.parseCmdLine() self.mergeValues() def initCfgDefault(self): ddd = { s_CommentKey: [ "The default config structure for mzn-test.py. ", s_ProgramDescr, "You can export this by --saveCfg and modify -> --mergeCfg,", "even having only partial JSON subtree in a merged file(s).", "Structure: COMMON_OPTIONS, SOLVER_/CHECKER_PROFILES, BACKEND_DEFS.", "Solver and checkers are selected from pre-defined profiles,", "which are in turn built from sequences of 'basic backend definitions'.", "Comments are either separate keys or added as list elements ('/// ...')", "in pre-selected positions; then, overriding items should keep that order." ], "COMMON_OPTIONS": { s_CommentKey: [ "'Solvers' and 'Checkers' select the profiles to use for solving and checking.", "At the moment only the 1st solver is used from Solvers." "The selected profiles must be known in SOLVER_PROFILES / CHECKER_PROFILES, resp." ], "Solvers": [ "MINIZINC", #"MZN-CPLEX", "/// At the moment only the 1st element is used for solving" ], "SOLUTION_CHECKING": { "Checkers": ["GECODE-CHK", "GUROBI-CHK", "CPLEX-CHK" ], "n_CheckedMax": [ -10, "/// Negative value means it's that many last solutions" ], "n_FailedSaveMax": [ 3, "/// After that many failed solutions, stop checking the instance" ], "s_FailedSaveFile": [ sFlnSolFailBase, "/// Filename to save failed solutions" ], }, "Instance_List": { s_CommentKey: [ "Params for instance lists.", "Instance list is a file containing an instance's model files,", "at most 1 instance per line.", "InstanceFileExt: only files with this extensions from a list file", "will be taken on each line" ], "InstanceFileExt": [".mzn", ".dzn"] ## Add json? TODO }, "runCommand": { "windows": { "runSilent": "echo \"WARNING. No timeout on Windows.\" & {2} 1>{3} 2>{4}", "runVerbose": "echo \"WARNING. No timeout on Windows.\" & {2} 3>&1 1>{3} 2>&3 | tee {4} & echo >>{4}" } ,"non-windows": { "runSilent": "ulimit -v {0}; timeout -k 1 {1} bash -c \"{2}\" 1>{3} 2>{4}", "runVerbose": "ulimit -v {0}; timeout -k 1 {1} bash -c \"{2}\" 3>&1 1>{3} 2>&3 | tee {4}; echo >>{4}" } } }, "SOLVER_PROFILES": { s_CommentKey: [ "Similar to CHECKER_PROFILES." ], "MINIZINC": [ "__BE_COMMON", "__BE_SOLVER", "BE_MINIZINC" ], "FZN-GUROBI": [ "__BE_COMMON", "__BE_SOLVER", "BE_FZN-GUROBI" ], "FZN-CPLEX": [ "__BE_COMMON", "__BE_SOLVER", "BE_FZN-CPLEX" ], "FZN-CBC": [ "__BE_COMMON", "__BE_SOLVER", "BE_FZN-CBC" ], "GUROBI": [ "__BE_COMMON", "__BE_SOLVER", "BE_GUROBI" ], "CPLEX": [ "__BE_COMMON", "__BE_SOLVER", "BE_CPLEX" ], "XPRESS": [ "__BE_COMMON", "__BE_SOLVER", "BE_XPRESS" ], "SCIP": [ "__BE_COMMON", "__BE_SOLVER", "BE_SCIP" ], "CBC": [ "__BE_COMMON", "__BE_SOLVER", "BE_CBC" ], "GECODE": [ "__BE_COMMON", "__BE_SOLVER", "BE_GECODE" ], "CHUFFED": [ "__BE_COMMON", "__BE_SOLVER", "BE_CHUFFED" ], "ORTOOLS": [ "__BE_COMMON", "__BE_SOLVER", "BE_ORTOOLS" ] #, # "MZN-GUROBI": [ "__BE_COMMON", "__BE_SOLVER", "BE_MZN-GUROBI" ], # "MZN-CPLEX": [ "__BE_COMMON", "__BE_SOLVER", "BE_MZN-CPLEX" ], # "MZN-CBC": [ "__BE_COMMON", "__BE_SOLVER", "BE_MZN-CBC" ], # "MZN-GECODE": [ "__BE_COMMON", "__BE_SOLVER", "BE_MZN-GECODE" ], # "FZN-CHUFFED": [ "__BE_COMMON", "__BE_SOLVER", "BE_FZN-CHUFFED" ] }, "CHECKER_PROFILES": { s_CommentKey: [ "Each profile gives a list of backend defs to use.", "Later backends in the list can override/add options, ", "for example for values to be read from the outputs.", "Adding is only possible if the key is prefixed by '"+s_AddKey+"'" ], "MINIZINC-CHK": [ "__BE_COMMON", "__BE_CHECKER_OLDMINIZINC", "BE_MINIZINC" ], "GECODE-CHK": [ "__BE_COMMON", "__BE_CHECKER", "BE_GECODE" ], "GUROBI-CHK": [ "__BE_COMMON", "__BE_CHECKER", "BE_GUROBI" ], "CPLEX-CHK": [ "__BE_COMMON", "__BE_CHECKER", "BE_CPLEX" ], "CHUFFED-CHK": [ "__BE_COMMON", "__BE_CHECKER", "BE_CHUFFED" ], "MZN-GECODE-CHK": [ "__BE_COMMON", "__BE_CHECKER", "BE_MZN-GECODE" ], "MZN-GUROBI-CHK": [ "__BE_COMMON", "__BE_CHECKER", "BE_MZN-GUROBI" ], "MZN-CPLEX-CHK": [ "__BE_COMMON", "__BE_CHECKER", "BE_MZN-CPLEX" ], "FZN-GECODE-CHK": [ "__BE_COMMON", "__BE_CHECKER_OLDMINIZINC", "BE_FZN-GECODE" ], "FZN-GECODE-SHELL-CHK": [ "__BE_COMMON", "__BE_CHECKER_OLDMINIZINC", "BE_FZN-GECODE_SHELL" ], "FZN-CHUFFED-CHK": [ "__BE_COMMON", "__BE_CHECKER", "BE_FZN-CHUFFED" ] }, "BACKEND_DEFS": { s_CommentKey: [ "__BE_COMMON initializes a basic backend structure.", "Each further backend in a profile list overrides or adds options." ], "__BE_COMMON": { s_CommentKey: [ "THE INITIALIZING BACKEND." ], "EXE": { s_CommentKey: [ "Solver call parameters" ], "s_SolverCall" : ["minizinc -v -s -a " + sDZNOutputAgrs + " %s", "/// The 1st element defines the call line. %s is replaced by the instance filename(s)."], "s_ExtraCmdline" : ["", "/// Only for __BE_SOLVER/__BE_CHECKER... subprofiles." " The 1st element gives extra cmdline arguments to the call"], "b_ThruShell" : [True, "/// Set True to call solver thru shell." " Then you can do shell tricks but Ctrl+C may not kill all subprocesses etc."], "n_TimeoutRealHard": [300, "/// Real-time timeout per instance, seconds," " for all solution steps together. Use mzn/backend options for CPU time limit."], "n_VMEMLIMIT_SoftHard": [8000000, 8000000, "/// 2 limits, soft/hard, in KB. Platform-dependent in Python 3.6. Default 8 GB"], }, "Stderr_Keylines": { s_CommentKey: [ "A complete line in stderr will be interpreted accordingly.", " Format: <outvar> : { <line>: <value>, ... }" " You can add own things here (use '"+s_AddKey+"' before new var name)", " which will be transferred into results" ] }, "Stderr_Keyvalues": { s_CommentKey: [ "Numerical values to be extracted from a line in stderr.", " { <outvar>: [ <regex search pattern>, <regex to replace by spaces>, <value's pos in the line>] }." ], ### The %%%mzn-stat values appear in stdout (as of May 2019) but leave them here just in case "Time_Flt": [ "%%%mzn-stat: flatTime", "[:=]", 3, "/// E.g., 'Flattening done, 3s' produces 3." " !!! This is interpreted as successful flattening by the checker" ], "ObjVal_Solver": [ "%%%mzn-stat objective=", "[,:/=]", 3, ## Need = to avoid mixup witht the bound "/// The objval as reported by solver."], "DualBnd_Solver": [ "%%%mzn-stat objectiveBound", "[,:/=]", 3 ], "CPUTime_Solver": [ "%%%mzn-stat solveTime", "[,:/=]", 3 ], "NNodes_Solver": [ "%%%mzn-stat nodes", "[,:/=]", 3 ], }, "Stdout_Keylines": { s_CommentKey: [ "Similar to Stderr_Keylines"], "Sol_Status": { "----------": 1, "==========": 2, "=====UNSATISFIABLE=====": -1, "=====UNBOUNDED=====": -2, "=====UNKNOWN=====": 0, "=====UNSATorUNBOUNDED=====": -3, "=====ERROR=====": -4 }, "Problem_Sense": { "%%%mzn-stat: method=\"maximize\"": 1, "%%%mzn-stat: method=\"minimize\"": -1, "%%%mzn-stat: method=\"satisfy\"": 0, } }, "Stdout_Keyvalues": { s_CommentKey: ["Similar to Stderr_Keyvalues." ], "Time_Flt": [ "%%%mzn-stat: flatTime", "[:=]", 3, "/// E.g., 'Flattening done, 3s' produces 3." " !!! This is interpreted as successful flattening by the checker" ], "ObjVal_Solver": [ "%%%mzn-stat objective=", "[,:/=]", 3, ## Need = to avoid mixup witht the bound "/// The objval as reported by solver."], "DualBnd_Solver": [ "%%%mzn-stat objectiveBound", "[,:/=]", 3 ], "CPUTime_Solver": [ "%%%mzn-stat solveTime", "[,:/=]", 3 ], "NNodes_Solver": [ "%%%mzn-stat nodes", "[,:/=]", 3 ], "ObjVal_MZN": [ "_objective", "[():=;%]", 2, "/// The objective value as evaluated by MZN." ], "RealTime_Solns2Out": [ "% time elapsed:", " ", 4 ], } }, "__BE_SOLVER": { s_CommentKey: ["Specializations for a general solver" ], "EXE": { "s_ExtraCmdline" : ["-a"], "b_ThruShell" : [True], "n_TimeoutRealHard": [300], # "n_VMEMLIMIT_SoftHard": [8000100000, 8100000000] } }, "__BE_CHECKER": { s_CommentKey: ["Specializations for a general checker" ], "EXE": { "s_ExtraCmdline" : [sFlatOptChecker], "b_ThruShell" : [True], "n_TimeoutRealHard": [15], # "n_VMEMLIMIT_SoftHard": [8100000000, 8100000000] } }, "__BE_CHECKER_OLDMINIZINC": { s_CommentKey: ["Specializations for a general checker using the 1.6 MiniZinc driver" ], "EXE": { "s_ExtraCmdline" : ["--mzn2fzn-cmd 'mzn2fzn -v -s --output-mode dzn " + sFlatOptChecker + "'"], "b_ThruShell" : [True], "n_TimeoutRealHard": [15], # "n_VMEMLIMIT_SoftHard": [8100000000, 8100000000] } }, "BE_MINIZINC": { s_CommentKey: [ "------------------- Specializations for pure minizinc driver" ], "EXE":{ "s_SolverCall": [ "minizinc --mzn2fzn-cmd 'mzn2fzn -v -s " + sDZNOutputAgrs + "' -s %s"], # _objective fails for checking "b_ThruShell" : [True], }, }, "BE_FZN-GUROBI": { s_CommentKey: [ "------------------- Specializations for Gurobi solver instance" ], "EXE":{ "s_SolverCall" : ["mzn2fzn -v -s -G linear " + sDZNOutputAgrs + " %s --fzn tmp.fzn --ozn tmp.ozn && mzn-gurobi -v -s tmp.fzn"], ## works without solns2out for now. Need thus when using shell call with system() TODO? #"opt_writeModel": ["--writeModel"] }, "Stderr_Keyvalues": { s_AddKey+"Preslv_Rows": [ "Presolved:", " ", 2 ], s_AddKey+"Preslv_Cols": [ "Presolved:", " ", 4 ], s_AddKey+"Preslv_Non0": [ "Presolved:", " ", 6 ] }, }, "BE_GUROBI": { s_CommentKey: [ "------------------- Specializations for Gurobi solver instance" ], "EXE":{ "s_SolverCall" : ["minizinc -v -s --solver gurobi --output-time " + sDZNOutputAgrs + " %s"], # _objective fails for checking TODO #"opt_writeModel": ["--writeModel"] }, "Stderr_Keyvalues": { s_AddKey+"Preslv_Rows": [ "Presolved:", " ", 2 ], s_AddKey+"Preslv_Cols": [ "Presolved:", " ", 4 ], s_AddKey+"Preslv_Non0": [ "Presolved:", " ", 6 ] }, }, "BE_MZN-GUROBI": { s_CommentKey: [ "------------------- Specializations for Gurobi solver instance" ], "EXE":{ "s_SolverCall" : ["mzn-gurobi -v -s -G linear --output-time " + sDZNOutputAgrs + " %s"], # _objective fails for checking TODO #"opt_writeModel": ["--writeModel"] }, "Stderr_Keyvalues": { s_AddKey+"Preslv_Rows": [ "Presolved:", " ", 2 ], s_AddKey+"Preslv_Cols": [ "Presolved:", " ", 4 ], s_AddKey+"Preslv_Non0": [ "Presolved:", " ", 6 ] }, }, "BE_FZN-CPLEX": { s_CommentKey: [ "------------------- Specializations for IBM ILOG CPLEX solver instance" ], "EXE": { "s_SolverCall" : ["mzn2fzn -v -s -G linear " + sDZNOutputAgrs + " %s --fzn tmp.fzn --ozn tmp.ozn && mzn-cplex -v -s tmp.fzn"] #"s_SolverCall" : ["./run-mzn-cplex.sh %s"], #"b_ThruShell" : [True], #"opt_writeModel": ["--writeModel"] }, "Stderr_Keyvalues": { s_AddKey+"Preslv_Rows": [ "Reduced MIP has [0-9]+ rows,", " ", 4 ], s_AddKey+"Preslv_Cols": [ "Reduced MIP has [0-9]+ rows,", " ", 6 ], s_AddKey+"Preslv_Non0": [ "Reduced MIP has [0-9]+ rows,", " ", 9 ] }, }, "BE_CPLEX": { s_CommentKey: [ "------------------- Specializations for IBM ILOG CPLEX solver instance" ], "EXE": { "s_SolverCall" : ["minizinc -v -s --solver cplex --output-time " + sDZNOutputAgrs + " %s"], # _objective fails for checking #"s_SolverCall" : ["./run-mzn-cplex.sh %s"], #"b_ThruShell" : [True], #"opt_writeModel": ["--writeModel"] }, "Stderr_Keyvalues": { s_AddKey+"Preslv_Rows": [ "Reduced MIP has [0-9]+ rows,", " ", 4 ], s_AddKey+"Preslv_Cols": [ "Reduced MIP has [0-9]+ rows,", " ", 6 ], s_AddKey+"Preslv_Non0": [ "Reduced MIP has [0-9]+ rows,", " ", 9 ] }, }, "BE_XPRESS": { s_CommentKey: [ "------------------- Specializations for FICO XPRESS solver instance" ], "EXE": { "s_SolverCall" : ["minizinc -v -s --solver xpress --output-time " + sDZNOutputAgrs + " %s"], # _objective fails for checking #"s_SolverCall" : ["./run-mzn-cplex.sh %s"], #"b_ThruShell" : [True], #"opt_writeModel": ["--writeModel"] }, "Stderr_Keyvalues": { ## Is different for XPRESS and -v fails at the moment anyway s_AddKey+"Preslv_Rows": [ "Reduced MIP has [0-9]+ rows,", " ", 4 ], s_AddKey+"Preslv_Cols": [ "Reduced MIP has [0-9]+ rows,", " ", 6 ], s_AddKey+"Preslv_Non0": [ "Reduced MIP has [0-9]+ rows,", " ", 9 ] }, }, "BE_MZN-CPLEX": { s_CommentKey: [ "------------------- Specializations for IBM ILOG CPLEX solver instance" ], "EXE": { "s_SolverCall" : ["mzn-cplex -v -s -G linear --output-time " + sDZNOutputAgrs + " %s"], # _objective fails for checking #"s_SolverCall" : ["./run-mzn-cplex.sh %s"], #"b_ThruShell" : [True], #"opt_writeModel": ["--writeModel"] }, "Stderr_Keyvalues": { s_AddKey+"Preslv_Rows": [ "Reduced MIP has [0-9]+ rows,", " ", 4 ], s_AddKey+"Preslv_Cols": [ "Reduced MIP has [0-9]+ rows,", " ", 6 ], s_AddKey+"Preslv_Non0": [ "Reduced MIP has [0-9]+ rows,", " ", 9 ] }, }, "BE_FZN-CBC": { s_CommentKey: [ "------------------- Specializations for COIN-OR Branch&Cut solver instance" ], "EXE": { "s_SolverCall" : ["mzn-cbc -v -s -G linear --output-time " + sDZNOutputAgrs + " %s --fzn tmp.fzn --ozn tmp.ozn && mzn-cplex -v -s tmp.fzn"], #"s_SolverCall" : ["./run-mzn-cplex.sh %s"], #"b_ThruShell" : [True], }, }, "BE_SCIP": { s_CommentKey: [ "------------------- Specializations for SCIP solver instance" ], "EXE": { "s_SolverCall" : ["minizinc -v -s --solver scip --output-time " + sDZNOutputAgrs + " %s"], # _objective fails for checking #"s_SolverCall" : ["./run-mzn-cplex.sh %s"], #"b_ThruShell" : [True], }, }, "BE_CBC": { s_CommentKey: [ "------------------- Specializations for COIN-OR Branch&Cut solver instance" ], "EXE": { "s_SolverCall" : ["minizinc -v -s --solver osicbc --output-time " + sDZNOutputAgrs + " %s"], # _objective fails for checking #"s_SolverCall" : ["./run-mzn-cplex.sh %s"], #"b_ThruShell" : [True], }, }, "BE_MZN-CBC": { s_CommentKey: [ "------------------- Specializations for COIN-OR Branch&Cut solver instance" ], "EXE": { "s_SolverCall" : ["mzn-cbc -v -s -G linear --output-time " + sDZNOutputAgrs + " %s"], # _objective fails for checking #"s_SolverCall" : ["./run-mzn-cplex.sh %s"], #"b_ThruShell" : [True], }, }, "BE_GECODE": { s_CommentKey: [ "------------------- Specializations for Gecode FlatZinc interpreter" ], "EXE": {# "s_SolverCall" : ["minizinc -s --solver gecode " + sDZNOutputAgrs + " %s"], # _objective fails for checking TODO "s_SolverCall" : ["minizinc -v -s --solver gecode " + sDZNOutputAgrs + " %s"], # --time 300000 "b_ThruShell" : [True], } }, "BE_MZN-GECODE": { s_CommentKey: [ "------------------- Specializations for Gecode FlatZinc interpreter" ], "EXE": {# "s_SolverCall" : ["mzn-fzn -s -G gecode --solver fzn-gecode " + sDZNOutputAgrs + " %s"], # _objective fails for checking TODO "s_SolverCall" : ["mzn-gecode -v -s -G gecode " + sDZNOutputAgrs + " %s"], # --time 300000 "b_ThruShell" : [True], } }, "BE_FZN-GECODE": { s_CommentKey: [ "------------------- Specializations for Gecode FlatZinc interpreter" ], "EXE": {# "s_SolverCall" : ["mzn-fzn -s -G gecode --solver fzn-gecode " + sDZNOutputAgrs + " %s"], # _objective fails for checking TODO "s_SolverCall" : ["minizinc -s -G gecode -f fzn-gecode --mzn2fzn-cmd 'mzn2fzn -v -s " + sDZNOutputAgrs + "' %s"], "b_ThruShell" : [True], } }, "BE_FZN-GECODE_SHELL": { s_CommentKey: [ "------------------- Specializations for Gecode FlatZinc interpreter" ], "EXE": {# "s_SolverCall" : ["mzn-fzn -s -G gecode --solver fzn-gecode " + sDZNOutputAgrs + " %s"], # _objective fails for checking TODO "s_SolverCall" : ["mzn2fzn -v -s -G gecode " + sDZNOutputAgrs + sFlatOptChecker + " %s --fzn tmp.fzn --ozn tmp.ozn && fzn-gecode tmp.fzn | solns2out tmp.ozn"], "b_ThruShell" : [True], "s_ExtraCmdline" : [""], } }, "BE_CHUFFED": { s_CommentKey: [ "------------------- Specializations for Chuffed FlatZinc interpreter" ], "EXE": { "s_SolverCall" : ["minizinc -v -s --solver chuffed -f --output-time " + sDZNOutputAgrs + " %s"], # _objective fails for checking } ## --fzn-flags --time-out --fzn-flags 300 } , "BE_ORTOOLS": { s_CommentKey: [ "------------------- Specializations for OR-Tools FlatZinc interpreter" ], "EXE": { "s_SolverCall" : ["minizinc -v -s --solver ortools -f --output-time " + sDZNOutputAgrs + " %s"], # _objective fails for checking } ## --fzn-flags --time-out --fzn-flags 300 } , "BE_FZN-CHUFFED": { s_CommentKey: [ "------------------- Specializations for Chuffed FlatZinc interpreter" ], "EXE": { "s_SolverCall" : ["mzn-fzn -v -s -G chuffed --solver fzn-chuffed --fzn-flags -f --output-time " + sDZNOutputAgrs + " %s"], # _objective fails for checking } ## --fzn-flags --time-out --fzn-flags 300 } } } return ddd ## self.nNoOptAndAtLeast2Feas = 0 ## Read a cfg file, instead of/in addition to the default cfg def mergeCfg(self, fln): ddd1 = None with open( fln, 'r' ) as rf: ddd1 = json.load( rf ) self.cfg = mergeJSON( self.cfg, ddd1 ) ## Merge cmdline values with cfg, performing some immediate actions ## And compile the backends constituting solver and checker(s) def mergeValues(self): if None!=self.args.mergeCfg: ### MERGE CFG FROM EXTRA FILES for eCfg in self.args.mergeCfg: self.mergeCfg( eCfg ) ################ Update some explicit cmdline params -- AFTER MERGING CFG FILES. if None!=self.args.failed: self.cfg["COMMON_OPTIONS"]["SOLUTION_CHECKING"]["s_FailedSaveFile"][0] = self.args.failed if None!=self.args.nCheckMax: self.cfg["COMMON_OPTIONS"]["SOLUTION_CHECKING"]["n_CheckedMax"][0] = self.args.nCheckMax if None!=self.args.nFailedSaveMax: self.cfg["COMMON_OPTIONS"]["SOLUTION_CHECKING"]["n_FailedSaveMax"][0] = self.args.nFailedSaveMax ################ SAVE FINAL CFG if None!=self.args.saveCfg: with utils.openFile_autoDir( self.args.saveCfg, 'w' ) as wf: print( "Saving final config to", self.args.saveCfg ) json.dump( self.cfg, wf, sort_keys=True, indent=json_config.n_JSON_Indent ) ### COMPILE THE SOLVER BACKEND if None!=self.args.solver: self.cfg["COMMON_OPTIONS"]["Solvers"][0] = self.args.solver slvPrfName = self.cfg["COMMON_OPTIONS"]["Solvers"][0] slvPrf = self.cfg["SOLVER_PROFILES"][slvPrfName] assert len(slvPrf)>0, "Solver profile '%s' should use at least a basic backend" % slvPrfName self.slvBE = self.cfg["BACKEND_DEFS"][slvPrf[0]] for i in range( 1, len( slvPrf ) ): self.slvBE = json_config.mergeJSON( self.slvBE, self.cfg["BACKEND_DEFS"][slvPrf[i]] ) if None!=self.args.tSolve: self.slvBE["EXE"]["n_TimeoutRealHard"][0] = self.args.tSolve assert None==self.args.solver or None==self.args.call, "ERROR: both solver call and a solver profile specified." if None!=self.args.call: ## After the compilation self.slvBE["EXE"]["s_SolverCall"][0] = self.args.call if None!=self.args.shellSolve: self.slvBE["EXE"]["b_ThruShell"][0] = self.args.shellSolve!=0 print ( "\nSolver/checker configurations:\n SLV_CFG: ", json.dumps( self.slvBE["EXE"] ) ) ### COMPILE THE CHECKER BACKENDS if None!=self.args.chkPrf and 0<len( self.args.chkPrf ): self.cfg["COMMON_OPTIONS"]["SOLUTION_CHECKING"]["Checkers"] = self.args.chkPrf chkPrfList = self.cfg["COMMON_OPTIONS"]["SOLUTION_CHECKING"]["Checkers"] self.chkBEs = [] for chkPrfName in chkPrfList: chkPrf = self.cfg["CHECKER_PROFILES"][chkPrfName] assert len(chkPrf)>0, "Checker profile '%s' should use at least a basic backend" % chkPrfName self.chkBEs.append( self.cfg["BACKEND_DEFS"][chkPrf[0]] ) for i in range( 1, len( chkPrf ) ): self.chkBEs[-1] = json_config.mergeJSON( self.chkBEs[-1], self.cfg["BACKEND_DEFS"][chkPrf[i]] ) if None!=self.args.tCheck: self.chkBEs[-1]["EXE"]["n_TimeoutRealHard"][0] = self.args.tCheck print ( " CHK_CFG: ", json.dumps( self.chkBEs[-1]["EXE"] ) ) print( "" ) ### SAVE THE SOLVER BACKEND if None!=self.args.saveSolverCfg: with utils.openFile_autoDir( self.args.saveSolverCfg, 'w' ) as wf: print( "Saving solver config to", self.args.saveSolverCfg ) json.dump( self.slvBE, wf, sort_keys=True, indent=json_config.n_JSON_Indent ) ### SAVE THE CHECKER BACKENDS if None!=self.args.saveCheckerCfg: with utils.openFile_autoDir( self.args.saveCheckerCfg, 'w' ) as wf: print( "Saving checker config to", self.args.saveCheckerCfg ) json.dump( self.chkBE, wf, sort_keys=True, indent=json_config.n_JSON_Indent ) self.sThisName = self.args.result if None!=self.args.name: self.sThisName = self.args.name def __init__(self): self.cfgDefault = self.initCfgDefault() self.cfg = self.cfgDefault ### Further parameters self.args = {} def __str__(self): s_Out = json.dumps( self.cfg, sort_keys=True, indent=json_config.n_JSON_Indent ) + '\n' s_Out += str(self.args) + '\n' s_Out += "\n >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> SOLVER BACKEND:\n" s_Out += json.dumps( self.slvBE, sort_keys=True, indent=json_config.n_JSON_Indent ) + '\n' s_Out += "\n >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> CHECKER BACKENDS:\n" for chkBE in self.chkBEs: s_Out += json.dumps( chkBE, sort_keys=True, indent=json_config.n_JSON_Indent ) + '\n' return s_Out
################################################################################################################# The MZNTest class##############################################################################################class MznTest:
## Produce a pair: first, a string identifying the instance which is given as a string of the instance files ## Second, same without paths and file extensions for short printing ## the elements are sorted according to the extensions list, then alphabetically def getIName( self, s_Inst ): lId = s_Inst.split() ## list of instance files lExt = self.params.cfg["COMMON_OPTIONS"]["Instance_List"]["InstanceFileExt"] lId = sorted( lId, key = lambda nm: ( lExt.index( os.path.splitext(nm)[1] ) \ if os.path.splitext(nm)[1] in lExt else len(lExt), os.path.basename( nm ) ) ) lId2 = [ os.path.splitext( os.path.basename( fln ) )[0] for fln in lId ]; return ' '.join(lId), ' '.join(lId2);
def obtainParams( self ): self.params.obtainParams()
## If an instance was specified in cmdline, or model list(s) supplied def compileExplicitModelLists(self): ## Get cmdline filenames or from the --instList arguments ## Can compile the list from log files, see below self.params.instList = [] ## Only if -l not used, take the pos args if (self.params.args.l_InstLists is None or 0==len( self.params.args.l_InstLists )) \ and self.params.args.instanceFiles is not None and 0<len( self.params.args.instanceFiles ): self.params.instList.append( " ".join( self.params.args.instanceFiles ) ) ## Mode "compare only" if (comparison lists and not run option) or no instances self.bCmpOnly = True if (not self.params.args.runAndCmp and (( \ self.params.args.compare is not None and 0<len(self.params.args.compare)) or \ ( self.params.args.l_InstLists is not None and 0<len( self.params.args.l_InstLists ) \ and self.params.args.instanceFiles is not None and 0<len( self.params.args.instanceFiles ) ))) \ else False ## If -l used, compile the inst list files if None!=self.params.args.l_InstLists and 0<len( self.params.args.l_InstLists ): for sFln in self.params.args.l_InstLists: with open( sFln, 'r' ) as rf: for line in rf: s_ModelFiles = "" for wrd in line.split(): fIsMF = False for ext in self.params.cfg["COMMON_OPTIONS"]["Instance_List"]["InstanceFileExt"]: if wrd.endswith( ext ): fIsMF = True break if fIsMF: s_ModelFiles += ' ' + wrd if 0<len( s_ModelFiles ): self.params.instList.append( s_ModelFiles ) ## Result logs can be used to extract the instance list, if wished def compileResultLogs(self): self.cmpRes = cmp_result_logs.CompareLogs() cmpFileList = [] if None!=self.params.args.compare: ### If -c used cmpFileList += self.params.args.compare ### If -l used, interpret pos arguments as comparison logs if None!=self.params.args.l_InstLists and 0<len( self.params.args.l_InstLists ): cmpFileList += self.params.args.instanceFiles for sFlnLog in cmpFileList: nEntries = 0 logCurrent, lLogNames = self.cmpRes.addLog( sFlnLog ) print( "Reading result log '", sFlnLog, "'... ", sep='', end='', flush=True ) with open( sFlnLog, 'r' ) as rf: while True: chJ = json_log.readLogJSONChunk( rf ) if None==chJ: break try: logCurrent[ self.getIName( chJ["Inst_Files"] )[0] ] = chJ if "TEST_NAME" in chJ: lLogNames[1] = chJ[ "TEST_NAME" ] print( " TEST NAME: '", chJ[ "TEST_NAME" ], "'... ", sep='', end='', flush=True ) except: print( "\n WARNING: unrecognized result chunk in file '", sFlnLog, "' before position", rf.tell(), ", doesn't contain all keys") else: nEntries += 1 print( len(logCurrent), "different instances among", nEntries, "recognized entries." ) def runTheInstances(self): logCurrent, lLogNames = self.cmpRes.addLog( self.params.args.result ) if self.params.sThisName!=lLogNames[0]: lLogNames[1] = self.params.sThisName ## TRUNCATING files first, then "a" - better on Win?? sNow = datetime.datetime.now().strftime("%Y%m%d_%H%M%S") self.sStartTime = sNow self.fileSol00 = utils.openFile_autoDir( self.params.args.resultUnchk, "w" ) self.fileSol = utils.openFile_autoDir(self.params.args.result.format(sNow + ("" if self.params.args.name is None else "__"+utils.flnfy(self.params.args.name))), "w" ) self.fileFailName = self.params.cfg["COMMON_OPTIONS"]["SOLUTION_CHECKING"]["s_FailedSaveFile"][0].format(sNow) self.fileFail = None ## Not opening yet self.nCheckedInstances = 0 self.nChecksFailed = 0 self.cmpRes.initListComparison() for i_Inst in range( len(self.params.instList) ): s_Inst = self.params.instList[ i_Inst ] self.initInstance( i_Inst, s_Inst ) self.solveOriginal( s_Inst ) try: if self.ifShouldCheck(): self.checkOriginal( s_Inst ) else: print( "NO CHECK, total check-failed instances:", self.nChecksFailed, "from", self.nCheckedInstances ) except: print( " WARNING: failed to check instance solution. ", sys.exc_info() ) ## Saving to the main log: self.saveSolution( self.fileSol ) ## Ranking: sSet_Inst = self.getIName( self.result["Inst_Files"] ) logCurrent[ sSet_Inst[0] ] = self.result try: self.cmpRes.compareInstance( sSet_Inst ) if i_Inst < len(self.params.instList)-1: print( "STATS: ", end='' ) self.cmpRes.summarizeCurrent( lLogNames ) print( "" ) except: print( " WARNING: failed to compare/rank instance. ", sys.exc_info() )
def compareLogs(self): self.cmpRes.initListComparison() theList = [ lfn for lfn in self.params.instList ] if 0==len( theList ): theList = self.cmpRes.getInstanceUnion() for sInst in theList: try: self.cmpRes.compareInstance( self.getIName( sInst ) ) except: print( " ------ WARNING: failed to compare/rank instance. ", ) traceback.print_exc() self.cmpRes.summarizeCmp() print( self.cmpRes.summarizeFinalHdr(), end='' )
def summarize(self): try: ### Printing summary print ('') print ('='*50) print(' SUMMARY') print ('='*50) stats = self.cmpRes.summarize() print (stats) except Exception as e: exc_type, exc_obj, exc_tb = sys.exc_info() fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1] print(exc_type, exc_obj, fname, exc_tb.tb_lineno) if not self.bCmpOnly: print( "Printing stats log to: ", end="" ) sFlnSL = sFlnStatLog.format( utils.flnfy( " ".join(self.params.args.l_InstLists if self.params.args.l_InstLists is not None else []) ) ) print( sFlnSL ) with utils.openFile_autoDir( sFlnSL, "a" ) as wf: wf.write( "\nRUN " + self.sStartTime + "--" + datetime.datetime.now().__str__() + ": " ) wf.write( sys.argv.__str__() ) wf.write( "\n" ) wf.write( stats ) print( "\nResult logs saved to '", self.params.args.result, "', with the unchecked log in '", self.params.args.resultUnchk, "'; failed solutions saved to '", self.params.cfg["COMMON_OPTIONS"]["SOLUTION_CHECKING"]["s_FailedSaveFile"][0], "' in an \"appendable JSON\" format, cf. https://github.com/pvorb/jsml." ## no more for shell: "\nSolver/checker stdout(err) outputs saved to '" + sDirResults + "/last_stdout(err)_...txt'." , sep='')
####################################################################################################################### MZNTest average-level #############################
## init def initInstance( self, i_Inst, s_Inst ): print( "\n--------------------- INSTANCE %d of %d: '" % (i_Inst+1, len( self.params.instList )), s_Inst, "' ----------------------", sep='' ) self.result = OrderedDict() if 0==i_Inst and None!=self.params.args.name: self.result["TEST_NAME"] = self.params.args.name self.result[ "Inst_Index_Of" ] = [ (i_Inst+1), len( self.params.instList ) ] self.result["Inst_Files"] = s_Inst self.solList = [] ## The solution list ## TODO: instance name source?
## Solve the original instance def solveOriginal( self, s_Inst ): self.result["__SOLVE__"] = self.solveInstance( s_Inst, self.params.slvBE, '_SOLVING', self.solList ) # print( " RESULT:\n", self.result ) self.saveSolution( self.fileSol00 ) ## Solve a given MZN instance and return the result map ## Arguments: instance files in a string, backend parameter dictionary ## slvName is used for screen output and last_std(out/err)_... naming, so better no spaces ## solList provided <=> this is solving (not checking) and will use --checkDZN if opted
## TODO ## Because we cannot limit memory of the subprocesses directly AND there seem to be bugs in the Python 3.4 impl, ## could replace the subprocess call by a call to an external which would run under given memory/time limits ## and save output to given files. ## OR: update from Python 3.4.2? ## NAming output / model files: sort input filenames, replace spaces/punctuation ### USE smth like ## keepcharacters = (' ','.','_') ## "".join(c for c in filename if c.isalnum() or c in keepcharacters else 'I').strip() ## PARAMETERS ## : solList: if not None, we are solving originally ( not checking ) def solveInstance(self, s_Inst, slvBE, slvName, solList=None): resSlv = OrderedDict() bChkDZN = True if None!=solList and None!=self.params.args.checkDZN else False if bChkDZN: print( "_PARSING '", self.params.args.checkDZN, sep='', end="'... " ) with open( self.params.args.checkDZN, 'r' ) as ro: mzn_exec.parseStdout( ro, resSlv, slvBE["Stdout_Keylines"], slvBE["Stdout_Keyvalues"], solList ) print( ro.tell(), "bytes", end='' ) if None!=self.params.args.checkStderr: print( " and '", self.params.args.checkStderr, sep='', end="'... " ) with open( self.params.args.checkStderr, 'r' ) as re: mzn_exec.parseStderr( re, resSlv, slvBE["Stderr_Keylines"], slvBE["Stderr_Keyvalues"] ) print( re.tell(), "bytes", end='' ) else: #### Solving oneself print( slvName, "... ", sep='', end='', flush=True ) s_Call = slvBE["EXE"]["s_SolverCall"][0] % s_Inst \ + ' ' + slvBE["EXE"]["s_ExtraCmdline"][0] if self.params.args.addOption is not None: for sOpt in self.params.args.addOption: s_Call += ' ' + sOpt if solList is not None: if self.params.args.addSolverOption is not None: for sOpt in self.params.args.addSolverOption: s_Call += ' ' + sOpt else: if self.params.args.addCheckerOption is not None: for sOpt in self.params.args.addCheckerOption: s_Call += ' ' + sOpt s_InstMerged = s_Inst.strip() ## s_InstMerged = regex.sub( r"[.\\/:~]", "", s_InstMerged ); ## s_InstMerged = regex.sub( r"[ ]", "-", s_InstMerged ); keepcharacters = ('-','_') s_InstMerged = "".join(c if c.isalnum() or c in keepcharacters else 'I' for c in s_InstMerged).strip() if solList is not None and self.params.args.useJoinedName is not None: for sUseJN in self.params.args.useJoinedName: s_UsingOpt = sUseJN % s_InstMerged s_Call += ' ' + s_UsingOpt if solList is not None: ## solving the original instance sFlnStdout = sFlnStdoutBase.format( s_InstMerged ) sFlnStderr = sFlnStderrBase.format( s_InstMerged ) if self.params.args.debug is not None and ( self.params.args.debug & 1 ): print( " CALL: \"", s_Call, "\"", sep='', flush=True ) else: sFlnStdout = 'last_stdout' + slvName + '.txt' sFlnStderr = 'last_stderr' + slvName + '.txt' if self.params.args.debug is not None and ( self.params.args.debug & 2 ): print( " CALL: \"", s_Call, "\"", sep='', flush=True ) resSlv["Solver_Call"] = s_Call resSlv["DateTime_Start"] = datetime.datetime.now().__str__() if 1==self.params.args.psutils: completed, tmAll = \ mzn_exec.runCmd( s_Call, slvBE["EXE"]["b_ThruShell"][0], slvBE["EXE"]["n_TimeoutRealHard"][0], slvBE["EXE"]["n_VMEMLIMIT_SoftHard"] ) with utils.openFile_autoDir( sFlnStdout, "w" ) as tf: tf.write( completed.stdout ) with utils.openFile_autoDir( sFlnStderr, "w" ) as tf: tf.write( completed.stderr ) print( "STDOUT/ERR: ", len(completed.stdout), '/', len(completed.stderr), " bytes", sep='', end=', ' ) mzn_exec.parseStderr( io.StringIO( completed.stderr ), resSlv, slvBE["Stderr_Keylines"], slvBE["Stderr_Keyvalues"] ) mzn_exec.parseStdout( io.StringIO( completed.stdout ), resSlv, slvBE["Stdout_Keylines"], slvBE["Stdout_Keyvalues"], solList ) ## Adding the outputs to the log ## resSlv["StdErr"] = completed.stderr ## resSlv["StdOut"] = completed.stdout else: ## use the 'system' call with utils.openFile_autoDir( sFlnStdout, "w" ) as tf: tf.write( "% EMPTY\n" ) with utils.openFile_autoDir( sFlnStderr, "w" ) as tf: tf.write( "% EMPTY" ) tmAll = mzn_exec.runCmdCmdline( s_Call, sFlnStdout, sFlnStderr, self.params.cfg["COMMON_OPTIONS"]["runCommand"], slvBE["EXE"]["n_TimeoutRealHard"][0], (self.params.args.verbose) if solList is not None else (self.params.args.vc), slvBE["EXE"]["n_VMEMLIMIT_SoftHard"] ) with open( sFlnStderr, "r" ) as rf: mzn_exec.parseStderr( rf, resSlv, slvBE["Stderr_Keylines"], slvBE["Stderr_Keyvalues"] ) with open( sFlnStdout, "r" ) as rf: mzn_exec.parseStdout( rf, resSlv, slvBE["Stdout_Keylines"], slvBE["Stdout_Keyvalues"], solList )
print( " t: {:.3f}".format( tmAll ), end=' s, ' ) resSlv["DateTime_Finish"] = datetime.datetime.now().__str__() resSlv["TimeReal_All"] = tmAll resSlv["TimeReal_LastStatus"] = 0 resSlv["Hostname"] = platform.uname()[1] ### For all cases, some postprocessing ############################# if "Sol_Status" not in resSlv: resSlv["Sol_Status"] = [-50, " !!!!! STATUS TOTALLY UNKNOWN - NO STATUS LINE PARSED."] if "Time_Flt" not in resSlv or utils.try_float( resSlv.get( "Time_Flt" ) ) is None: resSlv["NOFZN"] = [" !!!!! No flattening finish time registered or successfully parsed"] dTmLast = utils.try_float( resSlv.get( "RealTime_Solns2Out" ) ) if None!=dTmLast: resSlv["TimeReal_LastStatus"] = dTmLast / 1000.0 resSlv.pop( "RealTime_Solns2Out" ) ## if "SolutionLast" in resSlv: ## print( " SOLUTION_LAST:\n", resSlv["SolutionLast"], sep='' ) if None!=solList: print( " Nsol:", len(solList), end=',' ) print( " STATUS:", resSlv["Sol_Status"] ) return resSlv ## Append the unchecked solution of the instance to a temp. file def saveSolution(self, wf): json_log.writeLogChunk( wf, json.dumps( self.result, indent=json_config.n_JSON_Indent ) ) ## Return the necessity to check solutions. ## Can be false if we only want to compare different solvers. def ifShouldCheck(self): return \ 0<self.params.cfg["COMMON_OPTIONS"]["SOLUTION_CHECKING"]["n_FailedSaveMax"][0] and \ 0!=self.params.cfg["COMMON_OPTIONS"]["SOLUTION_CHECKING"]["n_CheckedMax"][0] and \ 0<len( self.params.chkBEs ) and \ self.ifCheckableStatus( self.result["__SOLVE__"]["Sol_Status"][0] ) ## def ifCheckableStatus( self, status ): return status>0 ## Check selected solutions of the instance def checkOriginal( self, s_Inst ): nCM = self.params.cfg["COMMON_OPTIONS"]["SOLUTION_CHECKING"]["n_CheckedMax"][0] nFSM = self.params.cfg["COMMON_OPTIONS"]["SOLUTION_CHECKING"]["n_FailedSaveMax"][0] assert 0!=nCM assert 0<len(self.solList) rng = None if 0<nCM: rng = range( 0, min( nCM, len(self.solList ) ) ) else: rng = range( -1, max( nCM-1, -len(self.solList)-1 ), -1 ) self.result["SOLUTION_CHECKS_DONE"] = 0 self.result["SOLUTION_CHECKS_FAILED"] = 0 fFailed = 0 ## Iterate over the solution range for iSol in rng: ## Try modify # self.solList[ iSol ] = self.solList[iSol][:20] + '1' + self.solList[iSol][21:] print ( "CHK SOL ", iSol if iSol<0 else (iSol+1), '/', len(rng), sep='', end='... ' ) with utils.openFile_autoDir( sFlnSolLastDzn, "w" ) as wf: ## Save the selected solution wf.write( self.solList[ iSol ] ) s_IC = s_Inst + ' ' + sFlnSolLastDzn bCheckOK = True chkFlSt = [] # self.result["__CHECKS__"] = [] for iChk in range ( len ( self.params.chkBEs ) ): chkBE = self.params.chkBEs[ iChk ] chkRes = self.solveInstance( s_IC, chkBE, '__Checker_'+str(iChk+1) ) # self.result["__CHECKS__"].append( chkRes ) if ( ## -51==chkRes["Sol_Status"][0] or ## NOFZN? No, flattener should report INFEAS. ( 0>chkRes["Sol_Status"][0] and -3<=chkRes["Sol_Status"][0] ) ): ## INFEAS bCheckOK = False chkFlSt = chkRes["Sol_Status"] self.result["SOLUTION_CHECKS_DONE"] += 1 if not bCheckOK: fFailed = 1 self.result["SOLUTION_CHECKS_FAILED"] += 1 self.result["SOLUTION_FAILED_LAST"] = self.solList[ iSol ] # self.result["SOLUTION_FAILED_LAST__CHKSTATUS"] = chkRes["Sol_Status"] if self.fileFail is None: self.fileFail = utils.openFile_autoDir( self.fileFailName, "w" ) self.saveSolution( self.fileFail ) if nFSM<=self.result["SOLUTION_CHECKS_FAILED"]: print ( self.result["SOLUTION_CHECKS_FAILED"], "failed solution(s) saved, go on" ) break self.nCheckedInstances += 1 self.nChecksFailed += fFailed print( " CHECK FAILS on this instance: ", self.result["SOLUTION_CHECKS_FAILED"], ", total check-failed instances: ", self.nChecksFailed, " from ", self.nCheckedInstances, sep='' ) def __init__(self): ## Default params self.params = MZT_Param()
####################################################################################################################### MZNTest public ########################################################################################################################### def run(self): self.obtainParams() self.compileExplicitModelLists() self.compileResultLogs() if not self.bCmpOnly: self.runTheInstances() else: self.compareLogs() self.summarize() # def mainmznTst = MznTest()mznTst.run()