diff --git a/CMake/alt.cmake b/CMake/alt.cmake index daf5e4e889df0ef4208880a4929683740ac58abf..a1d3a86413f8bcb380241008966df45db20773e0 100644 --- a/CMake/alt.cmake +++ b/CMake/alt.cmake @@ -37,11 +37,6 @@ function(alt_library name) ) endfunction() -function(alt_testing name) - cmake_parse_arguments(ARGUMENTS "" "" "DEPENDS;INCLUDES;TEST_INCLUDES;TEST_DEPENDS" ${ARGN}) - alt__unittests(${name} "${ARGUMENTS_TEST_DEPENDS}" "${ARGUMENTS_TEST_INCLUDES}") -endfunction() - function(alt__add_target name) cmake_parse_arguments(ARGUMENTS "" "" "DEPENDS;INCLUDES;TEST_INCLUDES;TEST_DEPENDS" ${ARGN}) # message("Registering target ${name} (dependencies: ${ARGUMENTS_DEPENDS}) (includes: ${ARGUMENTS_INCLUDES}) (test-includes: ${ARGUMENTS_TEST_INCLUDES})") diff --git a/CMakeLists.txt b/CMakeLists.txt index b330938982d402f35faa473d1a03ffa941164812..1fd4f3f86cc1431ade5c86b397c2aba388dab73f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -35,11 +35,8 @@ if(BUILD_TESTING) include(Catch) add_subdirectory(lib/catch2) include_directories(${PROJECT_SOURCE_DIR}/tests) - configure_file( - ${PROJECT_SOURCE_DIR}/CMake/configure_tests.hpp.in - ${PROJECT_BINARY_DIR}/configure_tests.hpp - ) - alt_module(alib2integrationtest) + + add_subdirectory(tests) endif() alt_module(alib2std) diff --git a/alib2integrationtest/CMakeLists.txt b/alib2integrationtest/CMakeLists.txt deleted file mode 100644 index ae59944254aa29e2d966d932e83ec8bc39405277..0000000000000000000000000000000000000000 --- a/alib2integrationtest/CMakeLists.txt +++ /dev/null @@ -1,5 +0,0 @@ -project(alt-testing VERSION ${CMAKE_PROJECT_VERSION}) - -alt_testing(alib2integrationtest - TEST_DEPENDS alib2cli alib2str alib2std alib2common alib2algo alib2data alib2elgo alib2aux stdc++fs catch2 -) diff --git a/alib2integrationtest/test-src/testing/TimeoutAqlTest.cpp b/alib2integrationtest/test-src/testing/TimeoutAqlTest.cpp deleted file mode 100644 index 34723e505c641630f3187fbf5fd1d16cac8b190a..0000000000000000000000000000000000000000 --- a/alib2integrationtest/test-src/testing/TimeoutAqlTest.cpp +++ /dev/null @@ -1,215 +0,0 @@ -#include "TimeoutAqlTest.hpp" -#include <cstring> -#include <exception> -#include <parser/Parser.h> - -#include <unistd.h> -#include <sys/wait.h> -#include <signal.h> -#include <string.h> - -#include <global/GlobalData.h> -#include <alib/exception> -#include <alib/random> - -#include <readline/StringLineInterface.h> - -#include <common/ResultInterpret.h> - -#define PIPE_RD 0 -#define PIPE_WR 1 - -#define FD_STDOUT 1 -#define FD_STDERR 2 - -/* Communication between signal handler and the rest of the program */ -int g_Wakeup [ 2 ]; // pipe for wakeup -int g_RecvSignal; // signalled flag - -std::string formatQueries ( const ext::vector < std::string > & queries ) { - std::string formated; - for ( const std::string & query : queries ) - formated += " " + query + "\n"; - return formated; -} - -int waitSignalTimeout ( int timeout ) { - struct timeval tv; - fd_set rd; - - tv.tv_sec = 0; - tv.tv_usec = timeout; - FD_ZERO ( &rd ); - FD_SET ( g_Wakeup [ PIPE_RD ], &rd ); - - select ( g_Wakeup [ PIPE_RD ] + 1, &rd, nullptr, nullptr, &tv ); - return g_RecvSignal; -} - - -std::string readFromFD ( int fd ) { - static const size_t BUFSIZE = 64; - - std::string res; - int rd; - char buf [ BUFSIZE ]; - - while ( ( rd = read ( fd, &buf, BUFSIZE - 1 ) ) > 0 ) { - res.append ( buf, rd ); - } - - return res; -} - -void writeToFD ( int fd, const std::string & message, const std::string & errorDesc ) { - if ( write ( fd, message.c_str ( ), message.length ( ) ) != ( ssize_t ) message.length ( ) ) - throw std::runtime_error ( "TimeoutAqlTest: write() to pipe failed (" + errorDesc + ")" ); -} - - -void newSigChild ( int ) { - g_RecvSignal = 1; - - // write into the pipe so select can read something, this effectively means that SIGCHILD was raised - writeToFD ( g_Wakeup [ PIPE_WR ], " ", "wakeup signalling" ); -} - -int aqlTest ( int fd_aql, const ext::vector < std::string > & queries, unsigned seed ) { - try { - cli::Environment environment; - - std::stringstream ss; - common::Streams::out = ext::reference_wrapper < std::ostream > ( ss ); - common::Streams::err = ext::reference_wrapper < std::ostream > ( ss ); - - // seed cli - cli::CharSequence sequence { cli::StringLineInterface ( "set seed " + ext::to_string ( seed ) ) }; - cli::Parser ( cli::Lexer ( std::move ( sequence ) ) ).parse ( ) -> run ( environment ); - - // run queries - for ( const std::string & q : queries ) { - cli::CharSequence querySequence { cli::StringLineInterface ( q ) }; - cli::Parser ( cli::Lexer ( std::move ( querySequence ) ) ).parse ( ) -> run ( environment ); - - writeToFD ( fd_aql, ss.str ( ), "writing aql output" ); - } - - return cli::ResultInterpret::cli ( environment.getResult ( ) ); /* 0 = OK */ - } catch ( const std::exception & ) { - std::ostringstream oss; - alib::ExceptionHandler::handle ( oss ); - writeToFD ( fd_aql, oss.str ( ), "writing aql output" ); - return -1; - } -} - -void _TimeoutAqlTest ( const std::chrono::microseconds & timeout, const ext::vector < std::string > & queries ) { - /* Register SIGCHLD handler */ - struct sigaction act; - memset ( &act, 0, sizeof ( act ) ); - act . sa_handler = newSigChild; - sigaction ( SIGCHLD, &act, nullptr ); - - int pipeAqlOutput [ 2 ]; /* parent-child communication ( aql output ) */ - int pipeStdout [ 2 ]; /* parent-child communication ( child stdout ) */ - int pipeStderr [ 2 ]; /* parent-child communication ( child stderr ) */ - - if ( pipe ( pipeAqlOutput ) != 0 ) - throw std::runtime_error ( "TimeoutAqlTest: Failed to initialize pipe (aql output)" ); - - if ( pipe ( pipeStdout ) != 0 ) - throw std::runtime_error ( "TimeoutAqlTest: Failed to initialize pipe (child stdout)" ); - - if ( pipe ( pipeStderr ) != 0 ) - throw std::runtime_error ( "TimeoutAqlTest: Failed to initialize pipe (child stderr)" ); - - /* SIGCHLD was not yet raised, initialize communication pipe */ - g_RecvSignal = 0; - if ( pipe ( g_Wakeup ) ) - throw std::runtime_error ( "TimeoutAqlTest: Failed to initialize pipe (wakeup signalling)" ); - - /* random seed for aql */ - unsigned seed = ext::random_devices::random ( ); - - /* do the forking */ - pid_t pid = fork (); - REQUIRE ( pid >= 0 ); - - if ( pid == 0 ) { /* child, run the test here */ - act . sa_handler = SIG_DFL; - sigaction ( SIGCHLD, &act, nullptr ); - - /* close unused ends of pipes in child */ - close ( g_Wakeup [ PIPE_RD ] ); - close ( g_Wakeup [ PIPE_WR ] ); - close ( pipeAqlOutput [ PIPE_RD ] ); - - close ( pipeStdout [ PIPE_RD ] ); - close ( pipeStderr [ PIPE_RD ] ); - - /* redirect stderr and stdout to pipe */ - dup2 ( pipeStdout [ PIPE_WR ], FD_STDOUT ); - dup2 ( pipeStderr [ PIPE_WR ], FD_STDERR ); - - /* run test */ - exit ( aqlTest ( pipeAqlOutput [ PIPE_WR ], queries, seed ) ); - } - - /* close unused ends of pipes in parent */ - close ( pipeAqlOutput [ PIPE_WR ] ); - close ( pipeStdout [ PIPE_WR ] ); - close ( pipeStderr [ PIPE_WR ] ); - - /* lets wait the specified time of microseconds, maybe the child will terminate on its own */ - if ( ! waitSignalTimeout ( timeout.count ( ) ) ) { - /* ... and in case it did not ... */ - kill ( pid, SIGTERM ); - while ( ! waitSignalTimeout ( 250000 ) ) /* 1/4 second */ - kill ( pid, SIGKILL ); - } - - /* child termination confirmed */ - int status; - waitpid ( pid, &status, 0 ); - close ( g_Wakeup [ PIPE_RD ] ); - close ( g_Wakeup [ PIPE_WR ] ); - - /* read child outputs */ - std::string childOutput [ 3 ] = { - readFromFD ( pipeAqlOutput [ PIPE_RD ] ), - readFromFD ( pipeStdout [ PIPE_RD ] ), - readFromFD ( pipeStderr [ PIPE_RD ] ) - }; - - /* communication is done */ - close ( pipeAqlOutput [ PIPE_RD ] ); - close ( pipeStdout [ PIPE_RD ] ); - close ( pipeStderr [ PIPE_RD ] ); - - /* determine test status */ - if ( WIFEXITED ( status ) ) { - INFO ( "AqlTest failure. Trying to execute: " << "\n" << formatQueries ( queries ) ); - INFO ( "Seed was: " << seed ); - INFO ( "Child aqlout was: >" << childOutput [ 0 ] << "<" ); - INFO ( "Child stdout was: >" << childOutput [ 1 ] << "<" ); - INFO ( "Child stderr was: >" << childOutput [ 2 ] << "<" ); - REQUIRE ( WEXITSTATUS ( status ) == 0 ); - } else if ( WIFSIGNALED ( status ) ) { - if ( WTERMSIG ( status ) == SIGTERM || WTERMSIG ( status ) == SIGKILL ) { /* killed by timeout control */ - WARN ( "AqlTest timeout (" << timeout.count ( ) << " us) reached. Trying to execute:\n" << formatQueries ( queries ) << - "Seed was: " << seed << "\n" << - "Child aqlout was: >" << childOutput [ 0 ] << "<\n" << - "Child stdout was: >" << childOutput [ 1 ] << "<\n" << - "Child stderr was: >" << childOutput [ 2 ] << "<\n" ); - CHECK_NOFAIL ( "timeout warning" == nullptr ); - } else { - INFO ( "AqlTest failure. Trying to execute: " << formatQueries ( queries ) ); - INFO ( "Seed was: " << seed ); - INFO ( "Child aqlout was: >" << childOutput [ 0 ] << "<" ); - INFO ( "Child stdout was: >" << childOutput [ 1 ] << "<" ); - INFO ( "Child stderr was: >" << childOutput [ 2 ] << "<" ); - INFO ( "Child process signaled, signal " << WTERMSIG ( status ) << " (" << strsignal ( WTERMSIG ( status ) ) << ")" ); - FAIL ( ); - } - } -} diff --git a/alib2integrationtest/test-src/testing/TimeoutAqlTest.hpp b/alib2integrationtest/test-src/testing/TimeoutAqlTest.hpp deleted file mode 100644 index ec27ec3570fa43973ae504655121d658ce3e4d1d..0000000000000000000000000000000000000000 --- a/alib2integrationtest/test-src/testing/TimeoutAqlTest.hpp +++ /dev/null @@ -1,17 +0,0 @@ -#ifndef _TIMEOUT_AQL_TEST_HPP__ -#define _TIMEOUT_AQL_TEST_HPP__ - -#include <catch2/catch.hpp> -#include <chrono> -#include <alib/vector> - -using namespace std::chrono_literals; - -void _TimeoutAqlTest ( const std::chrono::microseconds & timeout, const ext::vector < std::string > & queries ); - -template < class D > -void TimeoutAqlTest ( const D & timeout, const ext::vector < std::string > & queries ) { - _TimeoutAqlTest ( std::chrono::duration_cast < std::chrono::microseconds > ( timeout ), queries ); -} - -#endif /* _TIMEOUT_AQL_TEST_HPP__ */ diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt new file mode 100644 index 0000000000000000000000000000000000000000..a8d20cfe2a23a4999ce91df48cada700a6f40782 --- /dev/null +++ b/tests/CMakeLists.txt @@ -0,0 +1,82 @@ +# We have two kinds of tests: +# 1) aql/*.aql test files +# 2) static/*.cpp test files +# Both are supposed to be individual test executables + +configure_file( + ${CMAKE_CURRENT_SOURCE_DIR}/configure_tests.hpp.in + ${CMAKE_CURRENT_BINARY_DIR}/configure_tests.hpp + ) + +# Create base catch2 testing main +add_library(alt_testutils STATIC + testing/catch2_main.cpp + testing/AqlTest.hpp + testing/TimeoutAqlTest.cpp + testing/TimeoutAqlTest.hpp + testing/TestFiles.cpp + testing/TestFiles.hpp + testing/algorithms/UndefinedBehaviour.cpp + testing/algorithms/UndefinedBehaviour.h + testing/algorithms/Segfault.cpp + testing/algorithms/Segfault.h + ) +target_link_libraries(alt_testutils + PUBLIC catch2 + PUBLIC alib2cli alib2str alib2std alib2common alib2algo alib2data alib2elgo alib2aux + ) + +# 1) aql test files +# Catch2 does not allow to create TEST_CASE dynamically (and we want that -- +# mainly because of granularity in parallel processing) so we use this 'hack'. + +set(test_executable "test_aqltests") +add_executable(${test_executable} "") +target_link_libraries(${test_executable} + PUBLIC alt_testutils + ) + +# list all aql files +file(GLOB TEST_FILES_AQL + LIST_DIRECTORIES FALSE + RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} + CONFIGURE_DEPENDS + aql/*.aql + ) +foreach(AQL_FILEPATH IN LISTS TEST_FILES_AQL) + get_filename_component(AQL_FILENAME "${AQL_FILEPATH}" NAME_WE) + configure_file( + ${CMAKE_CURRENT_SOURCE_DIR}/aql.cpp.in + ${CMAKE_CURRENT_BINARY_DIR}/${AQL_FILENAME}.cpp + @ONLY + ) + set_source_files_properties(${CMAKE_CURRENT_BINARY_DIR}/${AQL_FILENAME}.cpp PROPERTIES GENERATED 1) + target_sources(${test_executable} PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/${AQL_FILENAME}.cpp) +endforeach() + +catch_discover_tests( + ${test_executable} + TEST_PREFIX "[aql test file]" + ) + + +# 2) cpp test files +set(test_executable "test_cppaqltests") +add_executable(${test_executable} "") +target_link_libraries(${test_executable} + PUBLIC alt_testutils + ) + +# list all cpp files +file(GLOB_RECURSE TEST_FILES_CPPAQL + LIST_DIRECTORIES FALSE + RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} + CONFIGURE_DEPENDS + cppaql/*.cpp cppaql/*.hpp cppaql/*.h + ) +target_sources(${test_executable} PRIVATE "${TEST_FILES_CPPAQL}") + +catch_discover_tests( + ${test_executable} + TEST_PREFIX "[cppaql test file]" + ) diff --git a/tests/aql.cpp.in b/tests/aql.cpp.in new file mode 100644 index 0000000000000000000000000000000000000000..7f60bd5aa1ff5736b101147ef09f936205b15e9f --- /dev/null +++ b/tests/aql.cpp.in @@ -0,0 +1,20 @@ +/** + * A template file for automatic testing of .aql files using Catch2. + * This template is processed by CMake's configure_file. For each + * .aql file a cpp file is generated. + * + * Author: Tomas Pecka + * Date: 3. 8. 2020 + **/ + +#include <catch2/catch.hpp> +#include <filesystem> +#include "configure_tests.hpp" +#include "testing/TimeoutAqlTest.hpp" + +using namespace std::literals; + +TEST_CASE("@AQL_FILEPATH@") +{ + TimeoutAqlTest ( 30s, std::filesystem::path( CMAKE_CURRENT_SOURCE_DIR ) / "@AQL_FILEPATH@"s, true ); +} diff --git a/tests/aql/eof.aql b/tests/aql/eof.aql new file mode 100644 index 0000000000000000000000000000000000000000..d25612d27c7dbe37369bba9b7811c03e810fe210 --- /dev/null +++ b/tests/aql/eof.aql @@ -0,0 +1,4 @@ +print 1 +print 2 +print 2 +quit 0 diff --git a/CMake/configure_tests.hpp.in b/tests/configure_tests.hpp.in similarity index 100% rename from CMake/configure_tests.hpp.in rename to tests/configure_tests.hpp.in diff --git a/alib2integrationtest/test-src/tests/approximateMatching.cpp b/tests/cppaql/approximateMatching.cpp similarity index 100% rename from alib2integrationtest/test-src/tests/approximateMatching.cpp rename to tests/cppaql/approximateMatching.cpp diff --git a/alib2integrationtest/test-src/tests/arbologyTest.cpp b/tests/cppaql/arbologyTest.cpp similarity index 100% rename from alib2integrationtest/test-src/tests/arbologyTest.cpp rename to tests/cppaql/arbologyTest.cpp diff --git a/alib2integrationtest/test-src/tests/borderArrayTest.cpp b/tests/cppaql/borderArrayTest.cpp similarity index 100% rename from alib2integrationtest/test-src/tests/borderArrayTest.cpp rename to tests/cppaql/borderArrayTest.cpp diff --git a/alib2integrationtest/test-src/tests/conversionsTest.cpp b/tests/cppaql/conversionsTest.cpp similarity index 100% rename from alib2integrationtest/test-src/tests/conversionsTest.cpp rename to tests/cppaql/conversionsTest.cpp diff --git a/alib2integrationtest/test-src/tests/conversionsTest2.cpp b/tests/cppaql/conversionsTest2.cpp similarity index 100% rename from alib2integrationtest/test-src/tests/conversionsTest2.cpp rename to tests/cppaql/conversionsTest2.cpp diff --git a/alib2integrationtest/test-src/tests/determinizeTest.cpp b/tests/cppaql/determinizeTest.cpp similarity index 100% rename from alib2integrationtest/test-src/tests/determinizeTest.cpp rename to tests/cppaql/determinizeTest.cpp diff --git a/alib2integrationtest/test-src/tests/dummyTest.cpp b/tests/cppaql/dummyTest.cpp similarity index 78% rename from alib2integrationtest/test-src/tests/dummyTest.cpp rename to tests/cppaql/dummyTest.cpp index 633f2c401159343fb3ff4b7b975d660770b7ec5f..06d9e8691b649e40ac5ca1760619f81699061b05 100644 --- a/alib2integrationtest/test-src/tests/dummyTest.cpp +++ b/tests/cppaql/dummyTest.cpp @@ -4,7 +4,7 @@ #include "testing/TimeoutAqlTest.hpp" #include "testing/TestFiles.hpp" -TEST_CASE ( "AQL Test", "[integration][dummy][!shouldfail][!hide]" ) { +TEST_CASE ( "AQL Test", "[integration][dummy][!shouldfail]" ) { ext::vector < std::string > qs = { "execute 1", "exec" @@ -48,7 +48,7 @@ TEST_CASE ( "Sanitizer Test", "[integration][dummy][!hide][!shouldfail]" ) { TimeoutAqlTest ( 1s, qs ); } -TEST_CASE ( "Failed Test", "[integration][dummy][!hide][!shouldfail]" ) { +TEST_CASE ( "Failed Test", "[integration][dummy][!shouldfail]" ) { ext::vector < std::string > qs = { "quit compare::PrimitiveCompare 1 2", }; @@ -56,12 +56,11 @@ TEST_CASE ( "Failed Test", "[integration][dummy][!hide][!shouldfail]" ) { TimeoutAqlTest ( 1s, qs ); } -TEST_CASE ( "Timeout Test", "[integration][dummy][!hide]" ) { - // we dont fail the tests (CHECK_NOFAIL) is used, so this is probably useless +TEST_CASE ( "Timeout Test", "[integration][dummy][!shouldfail]" ) { ext::vector < std::string > qs = { "execute \"generated some output\"", "execute \"generated some output for the second time\"", }; - TimeoutAqlTest ( 1us, qs ); + TimeoutAqlTest ( 1us, qs, true ); } diff --git a/alib2integrationtest/test-src/tests/exactMatching.cpp b/tests/cppaql/exactMatching.cpp similarity index 100% rename from alib2integrationtest/test-src/tests/exactMatching.cpp rename to tests/cppaql/exactMatching.cpp diff --git a/alib2integrationtest/test-src/tests/glushkovRteTest.cpp b/tests/cppaql/glushkovRteTest.cpp similarity index 100% rename from alib2integrationtest/test-src/tests/glushkovRteTest.cpp rename to tests/cppaql/glushkovRteTest.cpp diff --git a/alib2integrationtest/test-src/tests/glushkovRteTestGenerators.hpp b/tests/cppaql/glushkovRteTestGenerators.hpp similarity index 100% rename from alib2integrationtest/test-src/tests/glushkovRteTestGenerators.hpp rename to tests/cppaql/glushkovRteTestGenerators.hpp diff --git a/alib2integrationtest/test-src/tests/minimizeTest.cpp b/tests/cppaql/minimizeTest.cpp similarity index 100% rename from alib2integrationtest/test-src/tests/minimizeTest.cpp rename to tests/cppaql/minimizeTest.cpp diff --git a/alib2integrationtest/test-src/tests/normalizeTest.cpp b/tests/cppaql/normalizeTest.cpp similarity index 100% rename from alib2integrationtest/test-src/tests/normalizeTest.cpp rename to tests/cppaql/normalizeTest.cpp diff --git a/alib2integrationtest/test-src/tests/readerTest.cpp b/tests/cppaql/readerTest.cpp similarity index 100% rename from alib2integrationtest/test-src/tests/readerTest.cpp rename to tests/cppaql/readerTest.cpp diff --git a/alib2integrationtest/test-src/tests/regexpDerivationIntegralTest.cpp b/tests/cppaql/regexpDerivationIntegralTest.cpp similarity index 98% rename from alib2integrationtest/test-src/tests/regexpDerivationIntegralTest.cpp rename to tests/cppaql/regexpDerivationIntegralTest.cpp index 7f73871fd08a433a6a0fe3fb7472fb572f87c6a5..9738dfe49af3982d89826efed67f529d3c8586b0 100644 --- a/alib2integrationtest/test-src/tests/regexpDerivationIntegralTest.cpp +++ b/tests/cppaql/regexpDerivationIntegralTest.cpp @@ -23,7 +23,7 @@ TEST_CASE ( "RegExp Derivation/Integral test", "[integration]" ) { std::make_tuple ( Op::INTEGRAL, TestFiles::GetOne ( "/regexp/Melichar2-94.xml" ), TestFiles::GetOne ( "/regexp/Melichar2-94.i0.xml" ), "0" ), std::make_tuple ( Op::INTEGRAL, TestFiles::GetOne ( "/regexp/Melichar2-94.xml" ), TestFiles::GetOne ( "/regexp/Melichar2-94.i1.xml" ), "1" ) ); - ext::vector < std::string > qs = { + std::vector < std::string > qs = { "execute < " + std::get < 1 > ( definition ) + " > $regexp", "execute < " + std::get < 2 > ( definition ) + " > $result", "execute \"\\\"" + std::get < 3 > ( definition ) + "\\\"\"" + " | Move - | string::Parse @string::String - > $string", diff --git a/alib2integrationtest/test-src/tests/regexpOptimizeTest.cpp b/tests/cppaql/regexpOptimizeTest.cpp similarity index 100% rename from alib2integrationtest/test-src/tests/regexpOptimizeTest.cpp rename to tests/cppaql/regexpOptimizeTest.cpp diff --git a/alib2integrationtest/test-src/tests/treeNotationTest.cpp b/tests/cppaql/treeNotationTest.cpp similarity index 100% rename from alib2integrationtest/test-src/tests/treeNotationTest.cpp rename to tests/cppaql/treeNotationTest.cpp diff --git a/alib2integrationtest/test-src/tests/treeRepeatsTest.cpp b/tests/cppaql/treeRepeatsTest.cpp similarity index 100% rename from alib2integrationtest/test-src/tests/treeRepeatsTest.cpp rename to tests/cppaql/treeRepeatsTest.cpp diff --git a/tests/testing/AqlTest.hpp b/tests/testing/AqlTest.hpp new file mode 100644 index 0000000000000000000000000000000000000000..9f25e34f470cb8fe02bb71eacc195a744ea93e6e --- /dev/null +++ b/tests/testing/AqlTest.hpp @@ -0,0 +1,55 @@ +#ifndef AQL_TEST_HPP__ +#define AQL_TEST_HPP__ + +#include <istream> +#include <string> + +#include "alib/exception" +#include "common/ResultInterpret.h" +#include "global/GlobalData.h" +#include "environment/Environment.h" +#include "readline/IstreamLineInterface.h" +#include "readline/StringLineInterface.h" + +struct AqlTestResult { + int retcode; + unsigned seed; + std::string output; +}; + +/** @brief Runs AQL test with code stored in stream */ +template < typename Stream > +AqlTestResult AqlTest ( Stream & is, unsigned seed ) { + try { + cli::Environment environment; + cli::CommandResult result; + + // Capture CLI output + std::ostringstream oss; + common::Streams::out = ext::reference_wrapper < std::ostream > ( oss ); + common::Streams::err = ext::reference_wrapper < std::ostream > ( oss ); + + // seed cli, run test file + auto testSeed = std::make_shared < cli::StringLineInterface > ( "set seed " + ext::to_string ( seed ) ); + auto testFile = std::make_shared < cli::IstreamLineInterface < Stream& > > ( is ); + result = environment.execute ( testSeed ); + result = environment.execute ( testFile ); + + int returnValue; + if ( result == cli::CommandResult::QUIT || result == cli::CommandResult::RETURN ) { + returnValue = cli::ResultInterpret::cli ( environment.getResult ( ) ); + } else if ( result == cli::CommandResult::EOT || result == cli::CommandResult::OK ) { + returnValue = 0; + } else { + returnValue = 4; + } + + return { returnValue, seed, oss.str ( ) }; + } catch ( const std::exception & ) { + std::ostringstream oss; + alib::ExceptionHandler::handle ( oss ); + return { -1, seed, oss.str ( ) }; + } +} + +#endif /* AQL_TEST_HPP__ */ diff --git a/alib2integrationtest/test-src/testing/TestFiles.cpp b/tests/testing/TestFiles.cpp similarity index 95% rename from alib2integrationtest/test-src/testing/TestFiles.cpp rename to tests/testing/TestFiles.cpp index 89214587f29fd7460f60801868cf0ce1c74d516c..8ee996e34baa31f3f116aa27604e1f8aef8c6a08 100644 --- a/alib2integrationtest/test-src/testing/TestFiles.cpp +++ b/tests/testing/TestFiles.cpp @@ -6,7 +6,7 @@ using namespace std::literals; -std::filesystem::path TestFiles::TEST_FILES_BASEDIR = std::filesystem::path ( CMAKE_CURRENT_SOURCE_DIR ) / "examples2"; +std::filesystem::path TestFiles::TEST_FILES_BASEDIR = std::filesystem::path ( CMAKE_CURRENT_SOURCE_DIR ) / ".." / "examples2"; std::vector < std::string > TestFiles::Get ( const std::string& regex ) { return Get ( std::regex ( regex ) ); diff --git a/alib2integrationtest/test-src/testing/TestFiles.hpp b/tests/testing/TestFiles.hpp similarity index 100% rename from alib2integrationtest/test-src/testing/TestFiles.hpp rename to tests/testing/TestFiles.hpp diff --git a/tests/testing/TimeoutAqlTest.cpp b/tests/testing/TimeoutAqlTest.cpp new file mode 100644 index 0000000000000000000000000000000000000000..38de2bb61eb9261cbb6817b331c06190db41edc9 --- /dev/null +++ b/tests/testing/TimeoutAqlTest.cpp @@ -0,0 +1,251 @@ +#include "TimeoutAqlTest.hpp" +#include <cstring> +#include <exception> +#include <fstream> +#include <signal.h> +#include <string.h> +#include <sys/wait.h> +#include <unistd.h> + +#include "alib/exception" +#include "alib/random" +#include "AqlTest.hpp" +#include "common/ResultInterpret.h" +#include "global/GlobalData.h" +#include "parser/Parser.h" +#include "readline/IstreamLineInterface.h" +#include "readline/StringLineInterface.h" + +#define PIPE_RD 0 +#define PIPE_WR 1 + +#define FD_STDOUT 1 +#define FD_STDERR 2 + +/* Communication between signal handler and the rest of the program */ +int g_Wakeup [ 2 ]; // pipe for wakeup +int g_RecvSignal; // signalled flag + +int waitSignalTimeout ( int timeout ) { + struct timeval tv; + fd_set rd; + + tv.tv_sec = 0; + tv.tv_usec = timeout; + FD_ZERO ( &rd ); + FD_SET ( g_Wakeup [ PIPE_RD ], &rd ); + + select ( g_Wakeup [ PIPE_RD ] + 1, &rd, nullptr, nullptr, &tv ); + return g_RecvSignal; +} + +std::string readFromFD ( int fd ) { + static const size_t BUFSIZE = 64; + + std::string res; + int rd; + char buf [ BUFSIZE ]; + + while ( ( rd = read ( fd, &buf, BUFSIZE - 1 ) ) > 0 ) { + res.append ( buf, rd ); + } + + return res; +} + +void writeToFD ( int fd, const std::string & message, const std::string & errorDesc ) { + if ( write ( fd, message.c_str ( ), message.length ( ) ) != ( ssize_t ) message.length ( ) ) + throw std::runtime_error ( "TimeoutAqlTest: write() to pipe failed (" + errorDesc + ")" ); +} + +void newSigChild ( int ) { + g_RecvSignal = 1; + + // write into the pipe so select can read something, this effectively means that SIGCHILD was raised + writeToFD ( g_Wakeup [ PIPE_WR ], " ", "wakeup signalling" ); +} + +struct ChildStatus { + int status; + unsigned seed; + std::string outAql, outStdout, outStderr; +}; + +ChildStatus _TimeoutAqlTestImpl ( const std::chrono::microseconds & timeout, std::istream& is ) { + /* Register SIGCHLD handler */ + struct sigaction act; + memset ( &act, 0, sizeof ( act ) ); + act . sa_handler = newSigChild; + sigaction ( SIGCHLD, &act, nullptr ); + + int pipeAqlOutput [ 2 ]; /* parent-child communication ( aql output ) */ + int pipeStdout [ 2 ]; /* parent-child communication ( child stdout ) */ + int pipeStderr [ 2 ]; /* parent-child communication ( child stderr ) */ + + if ( pipe ( pipeAqlOutput ) != 0 ) + throw std::runtime_error ( "TimeoutAqlTest: Failed to initialize pipe (aql output)" ); + + if ( pipe ( pipeStdout ) != 0 ) + throw std::runtime_error ( "TimeoutAqlTest: Failed to initialize pipe (child stdout)" ); + + if ( pipe ( pipeStderr ) != 0 ) + throw std::runtime_error ( "TimeoutAqlTest: Failed to initialize pipe (child stderr)" ); + + /* SIGCHLD was not yet raised, initialize communication pipe */ + g_RecvSignal = 0; + if ( pipe ( g_Wakeup ) ) + throw std::runtime_error ( "TimeoutAqlTest: Failed to initialize pipe (wakeup signalling)" ); + + /* random seed for aql */ + unsigned seed = ext::random_devices::random ( ); + + /* do the forking */ + pid_t pid = fork (); + REQUIRE ( pid >= 0 ); + + if ( pid == 0 ) { /* child, run the test here */ + act . sa_handler = SIG_DFL; + sigaction ( SIGCHLD, &act, nullptr ); + + /* close unused ends of pipes in child */ + close ( g_Wakeup [ PIPE_RD ] ); + close ( g_Wakeup [ PIPE_WR ] ); + close ( pipeAqlOutput [ PIPE_RD ] ); + + close ( pipeStdout [ PIPE_RD ] ); + close ( pipeStderr [ PIPE_RD ] ); + + /* redirect stderr and stdout to pipe */ + dup2 ( pipeStdout [ PIPE_WR ], FD_STDOUT ); + dup2 ( pipeStderr [ PIPE_WR ], FD_STDERR ); + + /* run test */ + AqlTestResult res = AqlTest ( is, seed ); + writeToFD ( pipeAqlOutput [ PIPE_WR ], res.output, "writing cli output" ); + exit ( res.retcode ); + } + + /* close unused ends of pipes in parent */ + close ( pipeAqlOutput [ PIPE_WR ] ); + close ( pipeStdout [ PIPE_WR ] ); + close ( pipeStderr [ PIPE_WR ] ); + + /* lets wait the specified time of microseconds, maybe the child will terminate on its own */ + if ( ! waitSignalTimeout ( timeout.count ( ) ) ) { + /* ... and in case it did not ... */ + kill ( pid, SIGTERM ); + while ( ! waitSignalTimeout ( 250000 ) ) /* 1/4 second */ + kill ( pid, SIGKILL ); + } + + /* child termination confirmed */ + ChildStatus status; + + status.seed = seed; + + waitpid ( pid, &status.status, 0 ); + close ( g_Wakeup [ PIPE_RD ] ); + close ( g_Wakeup [ PIPE_WR ] ); + + /* read child outputs */ + status.outAql = readFromFD ( pipeAqlOutput [ PIPE_RD ] ); + status.outStdout = readFromFD ( pipeStdout [ PIPE_RD ] ); + status.outStderr = readFromFD ( pipeStderr [ PIPE_RD ] ); + + /* communication is done */ + close ( pipeAqlOutput [ PIPE_RD ] ); + close ( pipeStdout [ PIPE_RD ] ); + close ( pipeStderr [ PIPE_RD ] ); + + return status; +} + +void processTestChildStatus ( const ChildStatus & status, const std::chrono::microseconds & timeout, bool timeoutError, const std::string & test ) { + enum class ChildTerminationStatus { + Signal, + Timeout, + OK, + Fail, + } terminationState; + + /* determine test result */ + if ( WIFSIGNALED ( status.status ) && ( WTERMSIG ( status.status ) == SIGTERM || WTERMSIG ( status.status ) == SIGKILL ) ) + terminationState = ChildTerminationStatus::Timeout; + else if ( WIFSIGNALED ( status.status ) && WTERMSIG ( status.status ) != SIGTERM && WTERMSIG ( status.status ) != SIGKILL ) + terminationState = ChildTerminationStatus::Signal; + else if ( WIFEXITED ( status.status ) && WEXITSTATUS ( status.status ) == 0 ) + terminationState = ChildTerminationStatus::OK; + else if ( WIFEXITED ( status.status ) && WEXITSTATUS ( status.status ) != 0 ) + terminationState = ChildTerminationStatus::Fail; + else + FAIL ( ); + + /* format message */ + std::ostringstream oss; + oss << "AqlTest: "; + if ( terminationState == ChildTerminationStatus::Timeout ) { + oss << "timeout (" << timeout.count ( ) << " us) reached"; + } else if ( terminationState == ChildTerminationStatus::Signal ) { + oss << "killed by signal " << WTERMSIG ( status.status ) << " (" << strsignal ( WTERMSIG ( status.status ) ) << ")"; + } else if ( terminationState == ChildTerminationStatus::OK) { + oss << "completed succesfully"; + } else if ( terminationState == ChildTerminationStatus::Fail ) { + oss << "completed unsuccessfully"; + } else { + FAIL ( ); + } + oss << "\n"; + oss << test << "\n"; + oss << "Seed was: " << status.seed << "\n"; + oss << "Child aqlout was: >" << status.outAql << "<\n"; + oss << "Child stdout was: >" << status.outStdout << "<\n"; + oss << "Child stderr was: >" << status.outStderr << "<\n"; + + /* process result */ + if ( terminationState == ChildTerminationStatus::Timeout ) { + WARN ( oss.str ( ) ); + if ( timeoutError ) { + FAIL ( ); + } else { + CHECK_NOFAIL ( "Timeout reached" ); + } + } else if ( terminationState == ChildTerminationStatus::Signal ) { + WARN ( oss.str ( ) ); + FAIL ( ); + } else if ( terminationState == ChildTerminationStatus::OK ) { + // pass + } else if ( terminationState == ChildTerminationStatus::Fail ) { + WARN ( oss.str ( ) ); + REQUIRE ( WEXITSTATUS ( status.status ) == 0 ); + } else { + FAIL ( ); + } +} + +std::string printTest ( const std::filesystem::path & file ) { + return "Trying to execute testfile " + std::string ( file ) + "\n"; +} + +std::string printTest ( const std::vector < std::string > & queries ) { + std::ostringstream oss; + oss << "Trying to execute queries:\n"; + for ( const std::string & query : queries ) + oss << " " << query << "\n"; + return oss.str ( ); +} + +void _TimeoutAqlTest ( const std::chrono::microseconds & timeout, const std::filesystem::path & file, bool timeoutError ) { + std::ifstream ifs ( file ); + REQUIRE ( ifs.is_open ( ) ); + auto testChildStatus = _TimeoutAqlTestImpl ( timeout, ifs ); + processTestChildStatus ( testChildStatus, timeout, timeoutError, printTest ( file ) ); +} + +void _TimeoutAqlTest ( const std::chrono::microseconds & timeout, const std::vector < std::string > & queries, bool timeoutError ) { + std::stringstream ifs; + for ( const auto & q : queries ) + ifs << q << "\n"; + + auto testChildStatus = _TimeoutAqlTestImpl ( timeout, ifs ); + processTestChildStatus ( testChildStatus, timeout, timeoutError, printTest ( queries ) ); +} diff --git a/tests/testing/TimeoutAqlTest.hpp b/tests/testing/TimeoutAqlTest.hpp new file mode 100644 index 0000000000000000000000000000000000000000..217f41eff4755b08acb59a1769fd009635badc4e --- /dev/null +++ b/tests/testing/TimeoutAqlTest.hpp @@ -0,0 +1,34 @@ +#ifndef _TIMEOUT_AQL_TEST_HPP__ +#define _TIMEOUT_AQL_TEST_HPP__ + +#include <catch2/catch.hpp> +#include <chrono> +#include <filesystem> +#include <fstream> + +using namespace std::literals::chrono_literals; + +void _TimeoutAqlTest ( const std::chrono::microseconds & timeout, const std::filesystem::path & file, bool timeoutError ); +void _TimeoutAqlTest ( const std::chrono::microseconds & timeout, const std::vector < std::string > & queries, bool timeoutError ); + +/** + * @param timeout timeout (use std chrono literals) ofthe test + * @param queries list of queries to execute + * @param timeoutError is timeout an error? + */ +template < class D > +void TimeoutAqlTest ( const D & timeout, const std::vector < std::string > & queries, bool timeoutError = false ) { + _TimeoutAqlTest ( std::chrono::duration_cast < std::chrono::microseconds > ( timeout ), queries, timeoutError ); +} + +/** + * @param timeout timeout (use std chrono literals) ofthe test + * @param file path to file to test + * @param timeoutError is timeout an error? + */ +template < class D > +void TimeoutAqlTest ( const D & timeout, const std::filesystem::path & file, bool timeoutError ) { + _TimeoutAqlTest ( std::chrono::duration_cast < std::chrono::microseconds > ( timeout ), file, timeoutError ); +} + +#endif /* _TIMEOUT_AQL_TEST_HPP__ */ diff --git a/alib2integrationtest/test-src/algorithms/Segfault.cpp b/tests/testing/algorithms/Segfault.cpp similarity index 100% rename from alib2integrationtest/test-src/algorithms/Segfault.cpp rename to tests/testing/algorithms/Segfault.cpp diff --git a/alib2integrationtest/test-src/algorithms/Segfault.h b/tests/testing/algorithms/Segfault.h similarity index 100% rename from alib2integrationtest/test-src/algorithms/Segfault.h rename to tests/testing/algorithms/Segfault.h diff --git a/alib2integrationtest/test-src/algorithms/UndefinedBehaviour.cpp b/tests/testing/algorithms/UndefinedBehaviour.cpp similarity index 100% rename from alib2integrationtest/test-src/algorithms/UndefinedBehaviour.cpp rename to tests/testing/algorithms/UndefinedBehaviour.cpp diff --git a/alib2integrationtest/test-src/algorithms/UndefinedBehaviour.h b/tests/testing/algorithms/UndefinedBehaviour.h similarity index 100% rename from alib2integrationtest/test-src/algorithms/UndefinedBehaviour.h rename to tests/testing/algorithms/UndefinedBehaviour.h diff --git a/alib2integrationtest/test-src/main.cpp b/tests/testing/catch2_main.cpp similarity index 100% rename from alib2integrationtest/test-src/main.cpp rename to tests/testing/catch2_main.cpp