⟦9cb20bf5a⟧

TextFile

with Activity_Utilities;
package Regression is

    -- TEST SUBSYSTEM STRUCTURE ------------------------------------------
    --
    -- This package supports regression testing in subsystems that are
    -- structured like this:
    --
    --      Mumble.Sample_Tests        : Subsystem
    --         .RevX_Y_n               : Load View
    --            .Units               : Directory
    --               .New_Test_Results : Directory
    --               .Old_Test_Results : Directory
    --         .Test_Area              : World
    --
    -- There is nothing "special" about test views; they just contain
    -- "new" and "old" test results directories that facilitate automatic
    -- results comparisons.  The Test_Area world is created dynamically
    -- as required by executing tests.


    -- CONSTANTS ---------------------------------------------------------

    Test_Subsystem : constant String := Activity_Utilities.Current_Subsystem;
    Test_View : constant String := Activity_Utilities.Current_View;

    -- These constants are defined based on the current context when this
    -- package is elaborated, thus forcing tests to be run from within
    -- their enclosing view.  They will take on full path string values
    -- like !REGRESSION.MY_TESTS and !REGRESSION.MY_TESTS.REV1_0_0,
    -- respectively.


    -- EXCEPTIONS --------------------------------------------------------

    Test_Aborted : exception;

    -- This exception can be raised to terminate a test and give it a
    -- status of FAIL.


    -- TEST LOGGING ------------------------------------------------------

    procedure Begin_Test_Log;

    -- Begin_Test_Log should be invoked once at the beginning of a test
    -- set before any of the other logging procedures are invoked.
    -- It produces the header at the beginning of the test log.

    procedure Comment (Message : String);

    procedure Failed (Message : String; Abort_Test : Boolean := False);

    procedure Assert (Hypothesis : Boolean;
                      Message : String;
                      Abort_Test : Boolean := False);

    -- Comment and Failed write the given message to the test log
    -- (Current_Output).  Failed also changes the test status to failed,
    -- and can be instructed to abort the test immediately.  Assert
    -- is equivalent to Failed if the given hypothesis is false;
    -- otherwise it has no effect.

    procedure End_Test_Log;

    -- End_Test_Log should be invoked once at the end of a test set.
    -- It outputs a message indicating whether the tests as a group
    -- have passed or failed.

    -- Note that these report procedures write to a test log that is
    -- distinct from other test result files that are subject to
    -- automatic comparison.  Most test results should be directed
    -- to those files to take advantage of the automatic comparison.


    -- AUTOMATIC TEST RESULTS COMPARISON ---------------------------------

    procedure Compare (Filename : String;
                       Abort_Test : Boolean := False;
                       Options : String := "");

    -- Compares Test_Area.Filename with Old_Test_Results.Filename to verify
    -- that a test has produced the expected results.  If there are no
    -- differences between Old_Test_Results.Filename and Test_Area.Filename,
    -- then Test_Area.Filename is destroyed to save space.
    --
    -- If Old_Test_Results.Filename does not exist, or Test_Area.Filename
    -- does not match, then Test_Area.Filename is moved to the
    -- New_Test_Results directory.  Differences are summarized in
    -- New_Test_Results.Filename_Diff and the test is given the failed
    -- status.  The Abort_Test parameter determines if the test is
    -- allowed to continue after an unsuccessful comparison.
    --
    -- Note that by default the Test_Generic automatically compares the
    -- test's current output and error files, so testers need only compare
    -- any extra files that they choose to generate (usually none).
    -- These automatic comparisons can be suppressed when instantiating
    -- the Test_Generic.
    --
    -- The only Compare option is WILDCARDS, which says to interpret
    -- certain characters in Old_Test_Results.Filename as meta-characters
    -- for pattern matching.  (The wildcards supported here are the same
    -- as those documented for the File_Utilities package.)

    procedure Compare (New_Filename : String;
                       Old_Filename : String;
                       Abort_Test : Boolean := False;
                       Options : String := "");

    -- Occasionally, it may be necessary to use this more flexible
    -- version of Compare.  Both the New_Filename and Old_Filename are
    -- resolved in the current context utilizing normal naming conventions.
    -- This allows comparisons with files that have different names, new
    -- files that are not located in the Test_Area, and old files that are
    -- not located in the Old_Test_Results directory.
    --
    -- The test fails if either file cannot be found, or if the files do
    -- not match.  When the files do not match, differences are summarized
    -- in the New_Test_Results directory.  If the simple names of the files
    -- are the same, differences are placed in SimpleFilename_Diff, else
    -- in SimpleNewFilename_SimpleOldFilename_Diff.
    --
    -- This version of Compare supports two options, COPY_NEW and
    -- WILDCARDS.  If the COPY_NEW option is True (the default), the
    -- new file is copied to the New_Test_Results directory when the
    -- old file is not found and when the files do not match.  The
    -- WILDCARDS option behaves as above for meta-characters in
    -- Old_Filename.

    procedure Check_Image (Filename : String;
                           Abort_Test : Boolean := True;
                           Options : String := "");

    -- Check_Image is to interactive testing what Compare is to
    -- programmatic testing.  When an interactive test is first
    -- being logged, the tester can invoke Check_Image to save an
    -- image of the current window in the New_Test_Results directory.
    -- The Filename should be of the form TestId_ImageId, and
    -- should be carefully chosen to insure uniqueness and avoid
    -- accidental overlaying of previously checked images.
    --
    -- During subsequent executions of the test, windows are compared
    -- to their Old_Test_Results images.  If they fail to match, their
    -- images are left in the New_Test_Results directory and the test
    -- may be disposed of in three ways:
    --
    --   1) If the text file Mumble.Units.?.Fails_Allowed exists
    --      and contains a positive integer, this integer count will
    --      be decremented and the test will be permitted to continue.
    --      Otherwise, one of the following alternatives will be taken.
    --
    --   2) The test will be delayed until the tester can manually
    --      terminate it if the Abort_Test argument is False, or
    --
    --   3) the Test_Aborted exception will be raised (the default).
    --
    -- Note that if a portion of the current image is selected,
    -- only the selection will be compared, not the entire image.
    --
    -- The only Check_Image option, WILDCARDS, behaves as above.


    -- MISCELLANEOUS -----------------------------------------------------

    function Exists (Name : String) return Boolean;

    -- This function returns true if the given naming expression
    -- resolves to one or more objects in the directory system.


    -- GENERIC TEST PROCEDURE --------------------------------------------

    generic
        Test_Id : String;          -- test identifier, e.g., "FOO"
        with procedure Test_Body;  -- test proc to invoke, e.g., Foo_Test
        Options : String := "";    -- test options described below
    procedure Test_Generic;

    -- Instantiations of this Test_Generic do several things to prepare
    -- for a test:
    --
    --   1) Save the activities associated with the current job and
    --      session so that they can be restored after the test
    --      procedure is run.
    --
    --   2) Destroy any New_Test_Results.TestId_@ files that are
    --      leftover from the last time this test ran.
    --
    --   3) Establish the Test_Area world as the current context.
    --
    --   4) Redirect Current_Output and Current_Error to text files
    --      in the Test_Area world.
    --
    --   5) Invoke the test procedure.
    --
    -- Note that a specific response profile is not set.  If your output
    -- is sensitive to the response profile, you should set it in the
    -- Test_Body.
    --
    -- If any unhandled exceptions propagate out of the test procedure,
    -- the test is considered to have failed.  Otherwise, the test results
    -- are compared to their "correct" copies in the Old_Test_Results
    -- directory.  If the Old_Test_Results files do not exist, or if the
    -- files differ, the test is again considered to have failed.  The
    -- failed test results and a summary of any differences are placed in
    -- the New_Test_Results directory for further inspection and possible
    -- updating of the Old_Test_Results files.
    --
    -- The Test_Generic options (and their default values) are:
    --
    --   RECREATE_TEST_AREA (True)
    --     Recreate the Test_Area world even if it already exists.
    --     This ensures that the test will get a fresh Test_Area
    --     and tends to balance disk usage as tests run.  Setting
    --     this option to False reduces the test setup time and allows
    --     tests to pass information to each other in the Test_Area.
    --
    --   LINKS_IN_TEST_AREA (False)
    --     When recreating the Test_Area world, copy links from the
    --     test view.  By default, no links are copied.
    --
    --   COMPARE_OUTPUT (True)
    --     Automatically compare the test results that are written
    --     to the Current_Output.
    --
    --   COMPARE_ERROR (True)
    --     Automatically compare the test results that are written
    --     to the Current_Error.
    --
    --   WILDCARD_OUTPUT (False)
    --     Interpret wildcards when comparing Current_Output
    --     (implies Compare_Output=>True, and slows down comparison).
    --
    --   WILDCARD_ERROR (False)
    --     Interpret wildcards when comparing Current_Error
    --     (implies Compare_Error=>True, and slows down comparison).
    --
    --   PERFORMANCE_DATA (False)
    --     Add performance data (e.g., CPU time, disk I/O) to test log.

end Regression;
DataMuseum.dk

Rational R1000/400 Tapes

⟦9cb20bf5a⟧ TextFile

Derivation

TextFile
