testing: Move integration tests, add support for Aql files

Added support for testing aql files (tests/aql).
Removed alib2integrationtests/ in favour of tests/ directory.

Sorry for a messy commit.
Closes #198.
parent e918b583
......@@ -37,11 +37,6 @@ function(alt_library name)
)
endfunction()
function(alt_testing name)
cmake_parse_arguments(ARGUMENTS "" "" "DEPENDS;INCLUDES;TEST_INCLUDES;TEST_DEPENDS" ${ARGN})
alt__unittests(${name} "${ARGUMENTS_TEST_DEPENDS}" "${ARGUMENTS_TEST_INCLUDES}")
endfunction()
function(alt__add_target name)
cmake_parse_arguments(ARGUMENTS "" "" "DEPENDS;INCLUDES;TEST_INCLUDES;TEST_DEPENDS" ${ARGN})
# message("Registering target ${name} (dependencies: ${ARGUMENTS_DEPENDS}) (includes: ${ARGUMENTS_INCLUDES}) (test-includes: ${ARGUMENTS_TEST_INCLUDES})")
......
......@@ -35,11 +35,8 @@ if(BUILD_TESTING)
include(Catch)
add_subdirectory(lib/catch2)
include_directories(${PROJECT_SOURCE_DIR}/tests)
configure_file(
${PROJECT_SOURCE_DIR}/CMake/configure_tests.hpp.in
${PROJECT_BINARY_DIR}/configure_tests.hpp
)
alt_module(alib2integrationtest)
add_subdirectory(tests)
endif()
alt_module(alib2std)
......
project(alt-testing VERSION ${CMAKE_PROJECT_VERSION})
alt_testing(alib2integrationtest
TEST_DEPENDS alib2cli alib2str alib2std alib2common alib2algo alib2data alib2elgo alib2aux stdc++fs catch2
)
#ifndef _TIMEOUT_AQL_TEST_HPP__
#define _TIMEOUT_AQL_TEST_HPP__
#include <catch2/catch.hpp>
#include <chrono>
#include <alib/vector>
using namespace std::chrono_literals;
void _TimeoutAqlTest ( const std::chrono::microseconds & timeout, const ext::vector < std::string > & queries );
template < class D >
void TimeoutAqlTest ( const D & timeout, const ext::vector < std::string > & queries ) {
_TimeoutAqlTest ( std::chrono::duration_cast < std::chrono::microseconds > ( timeout ), queries );
}
#endif /* _TIMEOUT_AQL_TEST_HPP__ */
# We have two kinds of tests:
# 1) aql/*.aql test files
# 2) static/*.cpp test files
# Both are supposed to be individual test executables
configure_file(
${CMAKE_CURRENT_SOURCE_DIR}/configure_tests.hpp.in
${CMAKE_CURRENT_BINARY_DIR}/configure_tests.hpp
)
set(CMAKE_INCLUDE_CURRENT_DIR ON)
# Create base catch2 testing main
add_library(alt_testutils STATIC
testing/catch2_main.cpp
testing/AqlTest.hpp
testing/TimeoutAqlTest.cpp
testing/TimeoutAqlTest.hpp
testing/TestFiles.cpp
testing/TestFiles.hpp
testing/algorithms/UndefinedBehaviour.cpp
testing/algorithms/UndefinedBehaviour.h
testing/algorithms/Segfault.cpp
testing/algorithms/Segfault.h
)
target_link_libraries(alt_testutils
PUBLIC catch2
PUBLIC alib2cli alib2str alib2std alib2common alib2algo alib2data alib2elgo alib2aux
)
# 1) aql test files
# Catch2 does not allow to create TEST_CASE dynamically (and we want that --
# mainly because of granularity in parallel processing) so we use this 'hack'.
set(test_executable "test_aqltests")
add_executable(${test_executable} "")
target_link_libraries(${test_executable}
PUBLIC alt_testutils
)
# list all aql files
file(GLOB TEST_FILES_AQL
LIST_DIRECTORIES FALSE
RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
CONFIGURE_DEPENDS
aql/*.aql
)
foreach(AQL_FILEPATH IN LISTS TEST_FILES_AQL)
get_filename_component(AQL_FILENAME "${AQL_FILEPATH}" NAME_WE)
configure_file(
${CMAKE_CURRENT_SOURCE_DIR}/aql.cpp.in
${CMAKE_CURRENT_BINARY_DIR}/${AQL_FILENAME}.cpp
@ONLY
)
set_source_files_properties(${CMAKE_CURRENT_BINARY_DIR}/${AQL_FILENAME}.cpp PROPERTIES GENERATED 1)
target_sources(${test_executable} PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/${AQL_FILENAME}.cpp)
endforeach()
catch_discover_tests(
${test_executable}
TEST_PREFIX "[aql test file]"
)
# 2) cpp test files
set(test_executable "test_cppaqltests")
add_executable(${test_executable} "")
target_link_libraries(${test_executable}
PUBLIC alt_testutils
)
# list all cpp files
file(GLOB_RECURSE TEST_FILES_CPPAQL
LIST_DIRECTORIES FALSE
RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
CONFIGURE_DEPENDS
cppaql/*.cpp cppaql/*.hpp cppaql/*.h
)
target_sources(${test_executable} PRIVATE "${TEST_FILES_CPPAQL}")
catch_discover_tests(
${test_executable}
TEST_PREFIX "[cppaql test file]"
)
/**
* A template file for automatic testing of .aql files using Catch2.
* This template is processed by CMake's configure_file. For each
* .aql file a cpp file is generated.
*
* Author: Tomas Pecka
* Date: 3. 8. 2020
**/
#include <catch2/catch.hpp>
#include <filesystem>
#include "configure_tests.hpp"
#include "testing/TimeoutAqlTest.hpp"
using namespace std::literals;
TEST_CASE("@AQL_FILEPATH@")
{
TimeoutAqlTest ( 30s, std::filesystem::path( CMAKE_CURRENT_SOURCE_DIR ) / "@AQL_FILEPATH@"s, true );
}
print 1
print 2
print 2
quit 0
......@@ -4,7 +4,7 @@
#include "testing/TimeoutAqlTest.hpp"
#include "testing/TestFiles.hpp"
TEST_CASE ( "AQL Test", "[integration][dummy][!shouldfail][!hide]" ) {
TEST_CASE ( "AQL Test", "[integration][dummy][!shouldfail]" ) {
ext::vector < std::string > qs = {
"execute 1",
"exec"
......@@ -48,7 +48,7 @@ TEST_CASE ( "Sanitizer Test", "[integration][dummy][!hide][!shouldfail]" ) {
TimeoutAqlTest ( 1s, qs );
}
TEST_CASE ( "Failed Test", "[integration][dummy][!hide][!shouldfail]" ) {
TEST_CASE ( "Failed Test", "[integration][dummy][!shouldfail]" ) {
ext::vector < std::string > qs = {
"quit compare::PrimitiveCompare 1 2",
};
......@@ -56,12 +56,11 @@ TEST_CASE ( "Failed Test", "[integration][dummy][!hide][!shouldfail]" ) {
TimeoutAqlTest ( 1s, qs );
}
TEST_CASE ( "Timeout Test", "[integration][dummy][!hide]" ) {
// we dont fail the tests (CHECK_NOFAIL) is used, so this is probably useless
TEST_CASE ( "Timeout Test", "[integration][dummy][!shouldfail]" ) {
ext::vector < std::string > qs = {
"execute \"generated some output\"",
"execute \"generated some output for the second time\"",
};
TimeoutAqlTest ( 1us, qs );
TimeoutAqlTest ( 1us, qs, true );
}
......@@ -23,7 +23,7 @@ TEST_CASE ( "RegExp Derivation/Integral test", "[integration]" ) {
std::make_tuple ( Op::INTEGRAL, TestFiles::GetOne ( "/regexp/Melichar2-94.xml" ), TestFiles::GetOne ( "/regexp/Melichar2-94.i0.xml" ), "0" ),
std::make_tuple ( Op::INTEGRAL, TestFiles::GetOne ( "/regexp/Melichar2-94.xml" ), TestFiles::GetOne ( "/regexp/Melichar2-94.i1.xml" ), "1" ) );
ext::vector < std::string > qs = {
std::vector < std::string > qs = {
"execute < " + std::get < 1 > ( definition ) + " > $regexp",
"execute < " + std::get < 2 > ( definition ) + " > $result",
"execute \"\\\"" + std::get < 3 > ( definition ) + "\\\"\"" + " | Move - | string::Parse @string::String - > $string",
......
#ifndef AQL_TEST_HPP__
#define AQL_TEST_HPP__
#include <istream>
#include <string>
#include "alib/exception"
#include "common/ResultInterpret.h"
#include "global/GlobalData.h"
#include "environment/Environment.h"
#include "readline/IstreamLineInterface.h"
#include "readline/StringLineInterface.h"
struct AqlTestResult {
int retcode;
unsigned seed;
std::string output;
};
/** @brief Runs AQL test with code stored in stream */
template < typename Stream >
AqlTestResult AqlTest ( Stream & is, unsigned seed ) {
try {
cli::Environment environment;
cli::CommandResult result;
// Capture CLI output
std::ostringstream oss;
common::Streams::out = ext::reference_wrapper < std::ostream > ( oss );
common::Streams::err = ext::reference_wrapper < std::ostream > ( oss );
// seed cli, run test file
auto testSeed = std::make_shared < cli::StringLineInterface > ( "set seed " + ext::to_string ( seed ) );
auto testFile = std::make_shared < cli::IstreamLineInterface < Stream& > > ( is );
result = environment.execute ( testSeed );
result = environment.execute ( testFile );
int returnValue;
if ( result == cli::CommandResult::QUIT || result == cli::CommandResult::RETURN ) {
returnValue = cli::ResultInterpret::cli ( environment.getResult ( ) );
} else if ( result == cli::CommandResult::EOT || result == cli::CommandResult::OK ) {
returnValue = 0;
} else {
returnValue = 4;
}
return { returnValue, seed, oss.str ( ) };
} catch ( const std::exception & ) {
std::ostringstream oss;
alib::ExceptionHandler::handle ( oss );
return { -1, seed, oss.str ( ) };
}
}
#endif /* AQL_TEST_HPP__ */
......@@ -6,7 +6,7 @@
using namespace std::literals;
std::filesystem::path TestFiles::TEST_FILES_BASEDIR = std::filesystem::path ( CMAKE_CURRENT_SOURCE_DIR ) / "examples2";
std::filesystem::path TestFiles::TEST_FILES_BASEDIR = std::filesystem::path ( CMAKE_CURRENT_SOURCE_DIR ) / ".." / "examples2";
std::vector < std::string > TestFiles::Get ( const std::string& regex ) {
return Get ( std::regex ( regex ) );
......
#include "TimeoutAqlTest.hpp"
#include <cstring>
#include <exception>
#include <parser/Parser.h>
#include <unistd.h>
#include <sys/wait.h>
#include <fstream>
#include <signal.h>
#include <string.h>
#include <sys/wait.h>
#include <unistd.h>
#include <global/GlobalData.h>
#include <alib/exception>
#include <alib/random>
#include <readline/StringLineInterface.h>
#include <common/ResultInterpret.h>
#include "alib/exception"
#include "alib/random"
#include "AqlTest.hpp"
#include "common/ResultInterpret.h"
#include "global/GlobalData.h"
#include "parser/Parser.h"
#include "readline/IstreamLineInterface.h"
#include "readline/StringLineInterface.h"
#define PIPE_RD 0
#define PIPE_WR 1
......@@ -26,13 +26,6 @@
int g_Wakeup [ 2 ]; // pipe for wakeup
int g_RecvSignal; // signalled flag
std::string formatQueries ( const ext::vector < std::string > & queries ) {
std::string formated;
for ( const std::string & query : queries )
formated += " " + query + "\n";
return formated;
}
int waitSignalTimeout ( int timeout ) {
struct timeval tv;
fd_set rd;
......@@ -46,7 +39,6 @@ int waitSignalTimeout ( int timeout ) {
return g_RecvSignal;
}
std::string readFromFD ( int fd ) {
static const size_t BUFSIZE = 64;
......@@ -66,7 +58,6 @@ void writeToFD ( int fd, const std::string & message, const std::string & errorD
throw std::runtime_error ( "TimeoutAqlTest: write() to pipe failed (" + errorDesc + ")" );
}
void newSigChild ( int ) {
g_RecvSignal = 1;
......@@ -74,36 +65,13 @@ void newSigChild ( int ) {
writeToFD ( g_Wakeup [ PIPE_WR ], " ", "wakeup signalling" );
}
int aqlTest ( int fd_aql, const ext::vector < std::string > & queries, unsigned seed ) {
try {
cli::Environment environment;
std::stringstream ss;
common::Streams::out = ext::reference_wrapper < std::ostream > ( ss );
common::Streams::err = ext::reference_wrapper < std::ostream > ( ss );
// seed cli
cli::CharSequence sequence { cli::StringLineInterface ( "set seed " + ext::to_string ( seed ) ) };
cli::Parser ( cli::Lexer ( std::move ( sequence ) ) ).parse ( ) -> run ( environment );
// run queries
for ( const std::string & q : queries ) {
cli::CharSequence querySequence { cli::StringLineInterface ( q ) };
cli::Parser ( cli::Lexer ( std::move ( querySequence ) ) ).parse ( ) -> run ( environment );
writeToFD ( fd_aql, ss.str ( ), "writing aql output" );
}
return cli::ResultInterpret::cli ( environment.getResult ( ) ); /* 0 = OK */
} catch ( const std::exception & ) {
std::ostringstream oss;
alib::ExceptionHandler::handle ( oss );
writeToFD ( fd_aql, oss.str ( ), "writing aql output" );
return -1;
}
}
struct ChildStatus {
int status;
unsigned seed;
std::string outAql, outStdout, outStderr;
};
void _TimeoutAqlTest ( const std::chrono::microseconds & timeout, const ext::vector < std::string > & queries ) {
ChildStatus _TimeoutAqlTestImpl ( const std::chrono::microseconds & timeout, std::istream& is ) {
/* Register SIGCHLD handler */
struct sigaction act;
memset ( &act, 0, sizeof ( act ) );
......@@ -152,7 +120,9 @@ void _TimeoutAqlTest ( const std::chrono::microseconds & timeout, const ext::vec
dup2 ( pipeStderr [ PIPE_WR ], FD_STDERR );
/* run test */
exit ( aqlTest ( pipeAqlOutput [ PIPE_WR ], queries, seed ) );
AqlTestResult res = AqlTest ( is, seed );
writeToFD ( pipeAqlOutput [ PIPE_WR ], res.output, "writing cli output" );
exit ( res.retcode );
}
/* close unused ends of pipes in parent */
......@@ -169,47 +139,113 @@ void _TimeoutAqlTest ( const std::chrono::microseconds & timeout, const ext::vec
}
/* child termination confirmed */
int status;
waitpid ( pid, &status, 0 );
ChildStatus status;
status.seed = seed;
waitpid ( pid, &status.status, 0 );
close ( g_Wakeup [ PIPE_RD ] );
close ( g_Wakeup [ PIPE_WR ] );
/* read child outputs */
std::string childOutput [ 3 ] = {
readFromFD ( pipeAqlOutput [ PIPE_RD ] ),
readFromFD ( pipeStdout [ PIPE_RD ] ),
readFromFD ( pipeStderr [ PIPE_RD ] )
};
status.outAql = readFromFD ( pipeAqlOutput [ PIPE_RD ] );
status.outStdout = readFromFD ( pipeStdout [ PIPE_RD ] );
status.outStderr = readFromFD ( pipeStderr [ PIPE_RD ] );
/* communication is done */
close ( pipeAqlOutput [ PIPE_RD ] );
close ( pipeStdout [ PIPE_RD ] );
close ( pipeStderr [ PIPE_RD ] );
/* determine test status */
if ( WIFEXITED ( status ) ) {
INFO ( "AqlTest failure. Trying to execute: " << "\n" << formatQueries ( queries ) );
INFO ( "Seed was: " << seed );
INFO ( "Child aqlout was: >" << childOutput [ 0 ] << "<" );
INFO ( "Child stdout was: >" << childOutput [ 1 ] << "<" );
INFO ( "Child stderr was: >" << childOutput [ 2 ] << "<" );
REQUIRE ( WEXITSTATUS ( status ) == 0 );
} else if ( WIFSIGNALED ( status ) ) {
if ( WTERMSIG ( status ) == SIGTERM || WTERMSIG ( status ) == SIGKILL ) { /* killed by timeout control */
WARN ( "AqlTest timeout (" << timeout.count ( ) << " us) reached. Trying to execute:\n" << formatQueries ( queries ) <<
"Seed was: " << seed << "\n" <<
"Child aqlout was: >" << childOutput [ 0 ] << "<\n" <<
"Child stdout was: >" << childOutput [ 1 ] << "<\n" <<
"Child stderr was: >" << childOutput [ 2 ] << "<\n" );
CHECK_NOFAIL ( "timeout warning" == nullptr );
} else {
INFO ( "AqlTest failure. Trying to execute: " << formatQueries ( queries ) );
INFO ( "Seed was: " << seed );
INFO ( "Child aqlout was: >" << childOutput [ 0 ] << "<" );
INFO ( "Child stdout was: >" << childOutput [ 1 ] << "<" );
INFO ( "Child stderr was: >" << childOutput [ 2 ] << "<" );
INFO ( "Child process signaled, signal " << WTERMSIG ( status ) << " (" << strsignal ( WTERMSIG ( status ) ) << ")" );
return status;
}
void processTestChildStatus ( const ChildStatus & status, const std::chrono::microseconds & timeout, bool timeoutError, const std::string & test ) {
enum class ChildTerminationStatus {
Signal,
Timeout,
OK,
Fail,
} terminationState = ChildTerminationStatus::OK; /* make compiler happy */
/* determine test result */
if ( WIFSIGNALED ( status.status ) && ( WTERMSIG ( status.status ) == SIGTERM || WTERMSIG ( status.status ) == SIGKILL ) )
terminationState = ChildTerminationStatus::Timeout;
else if ( WIFSIGNALED ( status.status ) && WTERMSIG ( status.status ) != SIGTERM && WTERMSIG ( status.status ) != SIGKILL )
terminationState = ChildTerminationStatus::Signal;
else if ( WIFEXITED ( status.status ) && WEXITSTATUS ( status.status ) == 0 )
terminationState = ChildTerminationStatus::OK;
else if ( WIFEXITED ( status.status ) && WEXITSTATUS ( status.status ) != 0 )
terminationState = ChildTerminationStatus::Fail;
else
FAIL ( );
/* format message */
std::ostringstream oss;
oss << "AqlTest: ";
if ( terminationState == ChildTerminationStatus::Timeout ) {
oss << "timeout (" << timeout.count ( ) << " us) reached";
} else if ( terminationState == ChildTerminationStatus::Signal ) {
oss << "killed by signal " << WTERMSIG ( status.status ) << " (" << strsignal ( WTERMSIG ( status.status ) ) << ")";
} else if ( terminationState == ChildTerminationStatus::OK) {
oss << "completed succesfully";
} else if ( terminationState == ChildTerminationStatus::Fail ) {
oss << "completed unsuccessfully";
} else {
FAIL ( );
}
oss << "\n";
oss << test << "\n";
oss << "Seed was: " << status.seed << "\n";
oss << "Child aqlout was: >" << status.outAql << "<\n";
oss << "Child stdout was: >" << status.outStdout << "<\n";
oss << "Child stderr was: >" << status.outStderr << "<\n";
/* process result */
if ( terminationState == ChildTerminationStatus::Timeout ) {
WARN ( oss.str ( ) );
if ( timeoutError ) {
FAIL ( );
} else {
CHECK_NOFAIL ( "Timeout reached" );
}
} else if ( terminationState == ChildTerminationStatus::Signal ) {
WARN ( oss.str ( ) );
FAIL ( );
} else if ( terminationState == ChildTerminationStatus::OK ) {
// pass
} else if ( terminationState == ChildTerminationStatus::Fail ) {
WARN ( oss.str ( ) );
REQUIRE ( WEXITSTATUS ( status.status ) == 0 );
} else {
FAIL ( );
}
}
std::string printTest ( const std::filesystem::path & file ) {
return "Trying to execute testfile " + std::string ( file ) + "\n";
}
std::string printTest ( const std::vector < std::string > & queries ) {
std::ostringstream oss;
oss << "Trying to execute queries:\n";
for ( const std::string & query : queries )
oss << " " << query << "\n";
return oss.str ( );
}
void _TimeoutAqlTest ( const std::chrono::microseconds & timeout, const std::filesystem::path & file, bool timeoutError ) {
std::ifstream ifs ( file );
REQUIRE ( ifs.is_open ( ) );
auto testChildStatus = _TimeoutAqlTestImpl ( timeout, ifs );
processTestChildStatus ( testChildStatus, timeout, timeoutError, printTest ( file ) );
}
void _TimeoutAqlTest ( const std::chrono::microseconds & timeout, const std::vector < std::string > & queries, bool timeoutError ) {
std::stringstream ifs;
for ( const auto & q : queries )
ifs << q << "\n";
auto testChildStatus = _TimeoutAqlTestImpl ( timeout, ifs );
processTestChildStatus ( testChildStatus, timeout, timeoutError, printTest ( queries ) );
}
#ifndef _TIMEOUT_AQL_TEST_HPP__
#define _TIMEOUT_AQL_TEST_HPP__
#include <catch2/catch.hpp>
#include <chrono>
#include <filesystem>
#include <fstream>
using namespace std::literals::chrono_literals;
void _TimeoutAqlTest ( const std::chrono::microseconds & timeout, const std::filesystem::path & file, bool timeoutError );
void _TimeoutAqlTest ( const std::chrono::microseconds & timeout, const std::vector < std::string > & queries, bool timeoutError );
/**
* @param timeout timeout (use std chrono literals) ofthe test
* @param queries list of queries to execute
* @param timeoutError is timeout an error?
*/
template < class D >
void TimeoutAqlTest ( const D & timeout, const std::vector < std::string > & queries, bool timeoutError = false ) {
_TimeoutAqlTest ( std::chrono::duration_cast < std::chrono::microseconds > ( timeout ), queries, timeoutError );
}
/**
* @param timeout timeout (use std chrono literals) ofthe test
* @param file path to file to test
* @param timeoutError is timeout an error?
*/
template < class D >
void TimeoutAqlTest ( const D & timeout, const std::filesystem::path & file, bool timeoutError ) {
_TimeoutAqlTest ( std::chrono::duration_cast < std::chrono::microseconds > ( timeout ), file, timeoutError );
}
#endif /* _TIMEOUT_AQL_TEST_HPP__ */
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment