Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Bison 3.8 Parser #4062

Merged
merged 51 commits into from
Mar 19, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
51 commits
Select commit Hold shift + click to select a range
759d003
Derived Expressions Curl function with correctness test
lizdulac Jan 24, 2024
20bd7c8
Fixing index bug for internal boundaries
lizdulac Jan 26, 2024
a73f535
Curl Correctness Test Cleanup
lizdulac Jan 26, 2024
dd153a4
Remove debug statements in Test
lizdulac Feb 6, 2024
1a6c011
Merging Bison3.8 parser into ADIOS (ERRORS - cannot build)
lizdulac Feb 20, 2024
98997ca
CMake Tweaks
eisenhauer Feb 20, 2024
69f54f1
whoops, missed the cmake file.
eisenhauer Feb 21, 2024
3c25ccb
Updated parser merged with adios, updated error message for missing o…
lizdulac Feb 22, 2024
f0edbf3
Merge new parser with curl implementation
lizdulac Feb 22, 2024
9cec433
Clean up
lizdulac Feb 26, 2024
9e30fb5
Removed 'var' keyword from grammar and correctness test, and minor ch…
lizdulac Feb 28, 2024
390d44a
Merge branch 'curl' into Bison
lizdulac Feb 29, 2024
58fb934
Removed curl changes (only left Bison parser changes)
lizdulac Feb 29, 2024
2641043
run clang-format and exclude the pregen-source directory from clang-f…
eisenhauer Feb 29, 2024
9da7307
examples/derived shouldn't be included
eisenhauer Feb 29, 2024
3014923
shouldn't change examples/CMakeList
eisenhauer Feb 29, 2024
cfb131b
Merge branch 'master' into Bison
eisenhauer Mar 1, 2024
6e9ee47
Delete source/adios2/toolkit/derived/parser/pregen-source/Makefile
lizdulac Mar 1, 2024
ff27d47
Delete source/adios2/toolkit/derived/parser/CMakeLists.txt
lizdulac Mar 1, 2024
0dd4320
Delete source/adios2/toolkit/derived/parser/pregen-source/parser.output
lizdulac Mar 1, 2024
667716c
Kill redundancy in run-clang-format find command
eisenhauer Mar 1, 2024
c4855b6
remove binary
eisenhauer Mar 1, 2024
228b1ad
Fixing CI warnings - unused objects, class members initialized out of…
lizdulac Mar 1, 2024
ed49423
Work towards Windows
eisenhauer Mar 2, 2024
7ed2bc0
temporary fix for stdint issues
eisenhauer Mar 3, 2024
4c18d3f
Fixing CI build warnings: convert size_t to int, clearing an object o…
lizdulac Mar 4, 2024
de988ea
Formatting
lizdulac Mar 4, 2024
5f8cc22
Fixing CI warnings: ignore warnings on generated files
lizdulac Mar 6, 2024
1fad536
Windows
eisenhauer Mar 7, 2024
88e086a
clang-format
eisenhauer Mar 7, 2024
cab0730
Whoops, remove windows specific from general target
eisenhauer Mar 7, 2024
a01787b
Merge branch 'Bison' of https://github.com/lizdulac/ADIOS2 into Bison
eisenhauer Mar 7, 2024
fd87507
Win Tweaks
eisenhauer Mar 7, 2024
f20ee6a
limits.h
eisenhauer Mar 8, 2024
1c4eee0
Ignore IntelLLVM warning
eisenhauer Mar 11, 2024
ddd5a6a
Try init for ExpressionOperator
eisenhauer Mar 12, 2024
69b251e
Different tack
eisenhauer Mar 12, 2024
b235bc1
Add ASTNode and ASTDriver destructors, and implicitly call parse func…
lizdulac Mar 13, 2024
947d466
formatting
lizdulac Mar 13, 2024
30f7cc3
MinVarInfo unique_ptrs
lizdulac Mar 14, 2024
16256cb
formatting (again)
lizdulac Mar 14, 2024
b329e9f
Free ExprStr
eisenhauer Mar 14, 2024
c082411
Merge branch 'master' into Bison
eisenhauer Mar 14, 2024
3d22e5f
Change expression assignment character from : to =
lizdulac Mar 15, 2024
1082b80
Grammar accepts any blank space between alias definitions
lizdulac Mar 15, 2024
6d795bd
Restore Derived Variable status to defaulting to off
eisenhauer Mar 17, 2024
29b92fa
See if new test form works with Derived On
eisenhauer Mar 17, 2024
6d0e283
Always use derived_variable, even if not enabled
eisenhauer Mar 17, 2024
7616097
Whoops, accidently reverted the form of the derivation
eisenhauer Mar 17, 2024
ac372d2
Add definition
eisenhauer Mar 17, 2024
fcf5c13
Set Derived_Variable to default to OFF
eisenhauer Mar 18, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 15 additions & 0 deletions cmake/ADIOSBisonFlexSub.cmake
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
FUNCTION (SETUP_ADIOS_BISON_FLEX_SUB)

set (BISON_FLEX_PRECOMPILE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/toolkit/derived/parser/pregen-source")

ADD_CUSTOM_COMMAND(OUTPUT parser.cpp
COMMAND ${CMAKE_COMMAND} -E copy ${BISON_FLEX_PRECOMPILE_DIR}/parser.cpp ${CMAKE_CURRENT_BINARY_DIR}
COMMAND ${CMAKE_COMMAND} -E copy ${BISON_FLEX_PRECOMPILE_DIR}/parser.h ${CMAKE_CURRENT_BINARY_DIR}
COMMAND ${CMAKE_COMMAND} -E copy ${BISON_FLEX_PRECOMPILE_DIR}/location.hh ${CMAKE_CURRENT_BINARY_DIR}
COMMENT "Using pre-generated Bison Output from ${BISON_FLEX_PRECOMPILE_DIR}")
ADD_CUSTOM_COMMAND(OUTPUT lexer.cpp
COMMAND ${CMAKE_COMMAND} -E copy ${BISON_FLEX_PRECOMPILE_DIR}/lexer.cpp ${CMAKE_CURRENT_BINARY_DIR}
COMMENT "Using pre-generated Flex Output from ${BISON_FLEX_PRECOMPILE_DIR}")

set (BISON_Parser_OUTPUT_SOURCE parser.cpp PARENT_SCOPE)
ENDFUNCTION()
2 changes: 1 addition & 1 deletion scripts/ci/scripts/run-clang-format.sh
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ then
fi

# Check C and C++ code with clang-format
find source plugins testing examples bindings -regextype posix-extended -iregex '.*\.(h|c|cpp|tcc|cu)' | xargs clang-format -i
find source plugins testing examples bindings -regextype posix-extended -iregex '.*\.(h|c|cpp|tcc|cu)' ! -path "source/adios2/toolkit/derived/parser/pregen-source/*" | xargs clang-format -i
DIFF="$(git diff)"
if [ -n "${DIFF}" ]
then
Expand Down
43 changes: 41 additions & 2 deletions source/adios2/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -135,12 +135,51 @@ if (ADIOS2_HAVE_Derived_Variable)
toolkit/derived/Expression.cpp
toolkit/derived/Function.cpp toolkit/derived/Function.tcc
toolkit/derived/ExprHelper.h)
set_target_properties(adios2_core PROPERTIES
INCLUDE_DIRECTORIES "$<BUILD_INTERFACE:${ADIOS2_SOURCE_DIR}/source/adios2/toolkit/derived/parser>;$<BUILD_INTERFACE:${ADIOS2_BINARY_DIR}/source/adios2>")
find_package(BISON "3.8.2")
find_package(FLEX)

if(NOT BISON_FOUND OR NOT FLEX_FOUND)
include(ADIOSBisonFlexSub)
SETUP_ADIOS_BISON_FLEX_SUB()
else()
BISON_TARGET(MyParser
toolkit/derived/parser/parser.y
${CMAKE_CURRENT_BINARY_DIR}/parser.cpp
COMPILE_FLAGS "-o parser.cpp --header=parser.h"
DEFINES_FILE ${CMAKE_CURRENT_BINARY_DIR}/parser.h)
FLEX_TARGET(MyScanner
toolkit/derived/parser/lexer.l
COMPILE_FLAGS "-o lexer.cpp --header-file=lexer.h"
${CMAKE_CURRENT_BINARY_DIR}/lexer.cpp
DEFINES_FILE ${CMAKE_CURRENT_BINARY_DIR}/lexer.h)
ADD_FLEX_BISON_DEPENDENCY(MyScanner MyParser)
endif()
if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
SET_SOURCE_FILES_PROPERTIES(${CMAKE_CURRENT_BINARY_DIR}/lexer.cpp PROPERTIES COMPILE_FLAGS -Wno-sign-compare)
elseif(CMAKE_CXX_COMPILER_ID STREQUAL "Intel")
SET_SOURCE_FILES_PROPERTIES(${CMAKE_CURRENT_BINARY_DIR}/parser.cpp PROPERTIES COMPILE_FLAGS -Wno-unused-but-set-variable)
elseif(CMAKE_CXX_COMPILER_ID STREQUAL "IntelLLVM")
SET_SOURCE_FILES_PROPERTIES(${CMAKE_CURRENT_BINARY_DIR}/parser.cpp PROPERTIES COMPILE_FLAGS -Wno-unused-but-set-variable)
elseif(CMAKE_CXX_COMPILER_ID STREQUAL "MSVC")
SET_SOURCE_FILES_PROPERTIES(toolkit/derived/Expression.cpp toolkit/derived/Function.cpp PROPERTIES COMPILE_FLAGS "/wd4005 /wd4065 /wd4267 -DYY_NO_UNISTD_H")
endif()
add_library(adios2_core_derived
toolkit/derived/parser/lexer.cpp
toolkit/derived/parser/parser.cpp
${CMAKE_CURRENT_BINARY_DIR}/lexer.cpp
${CMAKE_CURRENT_BINARY_DIR}/parser.cpp
toolkit/derived/parser/ASTDriver.cpp
toolkit/derived/parser/ASTNode.cpp)
set_target_properties(adios2_core_derived PROPERTIES
VISIBILITY_INLINES_HIDDEN ON
INCLUDE_DIRECTORIES "$<BUILD_INTERFACE:${ADIOS2_SOURCE_DIR}/source/adios2/toolkit/derived/parser>;$<BUILD_INTERFACE:${ADIOS2_BINARY_DIR}/source/adios2>"
EXPORT_NAME core_derived
OUTPUT_NAME adios2${ADIOS2_LIBRARY_SUFFIX}_core_derived)
target_link_libraries(adios2_core PRIVATE adios2_core_derived)
set(maybe_adios2_core_derived adios2_core_derived)
if(CMAKE_CXX_COMPILER_ID STREQUAL "MSVC")
set_target_properties(adios2_core_derived PROPERTIES COMPILE_FLAGS "/wd4005 /wd4267 /wd4065 -DYY_NO_UNISTD_H")
endif()
endif()

set(maybe_adios2_core_cuda)
Expand Down
20 changes: 10 additions & 10 deletions source/adios2/core/VariableDerived.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ VariableDerived::VariableDerived(const std::string &name, adios2::derived::Expre
const DerivedVarType varType)
: VariableBase(name, exprType, helper::GetDataTypeSize(exprType), expr.GetShape(), expr.GetStart(),
expr.GetCount(), isConstant),
m_Expr(expr), m_DerivedType(varType)
m_DerivedType(varType), m_Expr(expr)
{
}

Expand All @@ -27,15 +27,15 @@ void VariableDerived::UpdateExprDim(std::map<std::string, std::tuple<Dims, Dims,
}

std::vector<std::tuple<void *, Dims, Dims>>
VariableDerived::ApplyExpression(std::map<std::string, MinVarInfo> NameToMVI)
VariableDerived::ApplyExpression(std::map<std::string, std::unique_ptr<MinVarInfo>> &NameToMVI)
{
size_t numBlocks = 0;
// check that all variables have the same number of blocks
for (auto variable : NameToMVI)
for (const auto &variable : NameToMVI)
{
if (numBlocks == 0)
numBlocks = variable.second.BlocksInfo.size();
if (numBlocks != variable.second.BlocksInfo.size())
numBlocks = variable.second->BlocksInfo.size();
if (numBlocks != variable.second->BlocksInfo.size())
helper::Throw<std::invalid_argument>("Core", "VariableDerived", "ApplyExpression",
" variables do not have the same number of blocks "
" in computing the derived variable " +
Expand All @@ -44,21 +44,21 @@ VariableDerived::ApplyExpression(std::map<std::string, MinVarInfo> NameToMVI)

std::map<std::string, std::vector<adios2::derived::DerivedData>> inputData;
// create the map between variable name and DerivedData object
for (auto variable : NameToMVI)
for (const auto &variable : NameToMVI)
{
// add the dimensions of all blocks into a vector
std::vector<adios2::derived::DerivedData> varData;
for (size_t i = 0; i < numBlocks; i++)
{
Dims start;
Dims count;
for (size_t d = 0; d < variable.second.Dims; d++)
for (int d = 0; d < variable.second->Dims; d++)
{
start.push_back(variable.second.BlocksInfo[i].Start[d]);
count.push_back(variable.second.BlocksInfo[i].Count[d]);
start.push_back(variable.second->BlocksInfo[i].Start[d]);
count.push_back(variable.second->BlocksInfo[i].Count[d]);
}
varData.push_back(adios2::derived::DerivedData(
{variable.second.BlocksInfo[i].BufferP, start, count}));
{variable.second->BlocksInfo[i].BufferP, start, count}));
}
inputData.insert({variable.first, varData});
}
Expand Down
2 changes: 1 addition & 1 deletion source/adios2/core/VariableDerived.h
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ class VariableDerived : public VariableBase
ApplyExpression(std::map<std::string, std::vector<void *>> NameToData,
std::map<std::string, std::tuple<Dims, Dims, Dims>> NameToDims);
std::vector<std::tuple<void *, Dims, Dims>>
ApplyExpression(std::map<std::string, MinVarInfo> mvi);
ApplyExpression(std::map<std::string, std::unique_ptr<MinVarInfo>> &mvi);
};

} // end namespace core
Expand Down
4 changes: 2 additions & 2 deletions source/adios2/engine/bp5/BP5Writer.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -516,7 +516,7 @@ void BP5Writer::ComputeDerivedVariables()
auto derivedVar = dynamic_cast<core::VariableDerived *>((*it).second.get());
std::vector<std::string> varList = derivedVar->VariableNameList();
// to create a mapping between variable name and the varInfo (dim and data pointer)
std::map<std::string, MinVarInfo> nameToVarInfo;
std::map<std::string, std::unique_ptr<MinVarInfo>> nameToVarInfo;
bool computeDerived = true;
for (auto varName : varList)
{
Expand All @@ -536,7 +536,7 @@ void BP5Writer::ComputeDerivedVariables()
std::cout << " .. skip derived variable " << (*it).second->m_Name << std::endl;
break;
}
nameToVarInfo.insert({varName, *mvi});
nameToVarInfo.insert({varName, std::unique_ptr<MinVarInfo>(mvi)});
}
// skip computing derived variables if it contains variables that are not written this step
if (!computeDerived)
Expand Down
2 changes: 1 addition & 1 deletion source/adios2/toolkit/derived/ExprHelper.h
Original file line number Diff line number Diff line change
Expand Up @@ -57,4 +57,4 @@ inline ExpressionOperator get_op(std::string op) { return string_to_op.at(op); }

}
}
#endif
#endif
42 changes: 29 additions & 13 deletions source/adios2/toolkit/derived/Expression.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -2,19 +2,35 @@
#define ADIOS2_DERIVED_Expression_CPP_

#include "Expression.h"
#include "parser/ASTNode.h"
#include "parser/parser.h"
#include "parser/ASTDriver.h"

namespace adios2
{
namespace detail
{
// helper function
adios2::detail::ExpressionOperator convert_op(std::string opname)
{
adios2::detail::ExpressionOperator op;
try
{
op = adios2::detail::get_op(opname);
}
catch (std::out_of_range &e)
{
(void)e; // use e
helper::Throw<std::invalid_argument>("Derived", "ExprHelper", "get_op",
"Parser cannot recognize operator '" + opname + "'.");
}
return op;
};

adios2::derived::ExpressionTree ASTNode_to_ExpressionTree(adios2::detail::ASTNode *node)
{
adios2::derived::ExpressionTree exprTree_node(node->operation);
for (adios2::detail::ASTNode *e : node->sub_expr)
adios2::derived::ExpressionTree exprTree_node(convert_op(node->get_opname()));
for (adios2::detail::ASTNode *e : node->get_subexprs())
{
switch (e->operation)
switch (convert_op(e->get_opname()))
{
case adios2::detail::ExpressionOperator::OP_ALIAS: // add variable given by alias
// add an index operation in the chain if the variable contains indeces
Expand All @@ -25,19 +41,19 @@ adios2::derived::ExpressionTree ASTNode_to_ExpressionTree(adios2::detail::ASTNod
index_expr.add_child(e->lookup_var_path(e->alias));
expTree_node->add_child(expr);
}*/
exprTree_node.add_child(e->lookup_var_path(e->alias));
exprTree_node.add_child(e->get_varname());
break;
case adios2::detail::ExpressionOperator::OP_PATH: // add variable name
exprTree_node.add_child(e->alias);
exprTree_node.add_child(e->get_varname());
break;
case adios2::detail::ExpressionOperator::OP_NUM: // set the base value for the operation
exprTree_node.set_base(e->value);
exprTree_node.set_base(e->get_value());
break;
default: // if the children nodes are other expressions, convert them to expressions
auto temp_node = ASTNode_to_ExpressionTree(e);
// move from a binary to a multinary tree if the child has the same operation
if (e->operation == node->operation &&
adios2::detail::op_property.at(e->operation).is_associative)
if (convert_op(e->get_opname()) == convert_op(node->get_opname()) &&
adios2::detail::op_property.at(convert_op(e->get_opname())).is_associative)
{
// concatenate exprTree with temp_node
for (std::tuple<adios2::derived::ExpressionTree, std::string, bool> childTree :
Expand All @@ -63,10 +79,10 @@ namespace derived
{

Expression::Expression(std::string string_exp)
: ExprString(string_exp), m_Shape({0}), m_Start({0}), m_Count({0})
: m_Shape({0}), m_Start({0}), m_Count({0}), ExprString(string_exp)
{
adios2::detail::ASTNode *root_node = adios2::detail::parse_expression(string_exp);
m_Expr = adios2::detail::ASTNode_to_ExpressionTree(root_node);
adios2::detail::ASTDriver drv(string_exp);
m_Expr = adios2::detail::ASTNode_to_ExpressionTree(drv.getAST());
}

std::vector<std::string> Expression::VariableNameList() { return m_Expr.VariableNameList(); }
Expand Down
2 changes: 1 addition & 1 deletion source/adios2/toolkit/derived/Expression.h
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ class ExpressionTree
std::vector<std::tuple<ExpressionTree, std::string, bool>> sub_exprs;
OpInfo detail;

ExpressionTree(){};
ExpressionTree() : detail({adios2::detail::ExpressionOperator::OP_NULL, {}, 0}) {}
ExpressionTree(adios2::detail::ExpressionOperator o) : detail({o, {}, 0}) {}
ExpressionTree(adios2::detail::ExpressionOperator o, double c) : detail({o, {}, 0}) {}
ExpressionTree(std::vector<std::tuple<size_t, size_t, size_t>> indices)
Expand Down
2 changes: 1 addition & 1 deletion source/adios2/toolkit/derived/Function.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@ DerivedData MagnitudeFunc(std::vector<DerivedData> inputData, DataType type)
T *magValues = ApplyOneToOne<T>(inputData, dataSize, [](T a, T b) { return a + b * b; }); \
for (size_t i = 0; i < dataSize; i++) \
{ \
magValues[i] = std::sqrt(magValues[i]); \
magValues[i] = (T)std::sqrt(magValues[i]); \
} \
return DerivedData({(void *)magValues, inputData[0].Start, inputData[0].Count}); \
}
Expand Down
2 changes: 1 addition & 1 deletion source/adios2/toolkit/derived/Function.tcc
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ T *ApplyOneToOne(std::vector<DerivedData> inputData, size_t dataSize,
std::cout << "Allocation failed for the derived data" << std::endl;
// TODO - throw an exception
}
memset(outValues, 0, dataSize * sizeof(T));
memset((void *)outValues, 0, dataSize * sizeof(T));
for (auto &variable : inputData)
{
for (size_t i = 0; i < dataSize; i++)
Expand Down
101 changes: 101 additions & 0 deletions source/adios2/toolkit/derived/parser/ASTDriver.cpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,101 @@
#include "ASTDriver.h"

namespace adios2
{
namespace detail
{

using indx_type = std::vector<std::tuple<int, int, int>>;

ASTDriver::ASTDriver() {}

ASTDriver::ASTDriver(const std::string input) { ASTDriver::parse(input); }

ASTDriver::~ASTDriver()
{
while (holding.size() > 0)
{
delete holding.top();
holding.pop();
}
}

ASTNode *ASTDriver::getAST()
{
// TODO: check only one ASTNode remains in holding
// else throw error that parsing failed
resolve(holding.top());
return holding.top();
}

void ASTDriver::resolve(ASTNode *node)
{
if (!node->get_alias().empty())
{
std::tuple<std::string, indx_type> var_info;
var_info = lookup_var(node->get_alias());
node->set_varname(std::get<0>(var_info));
node->set_indices(std::get<1>(var_info));
}
for (ASTNode *subexpr : node->get_subexprs())
{
resolve(subexpr);
}
}

std::tuple<std::string, indx_type> ASTDriver::lookup_var(const std::string alias)
{
return aliases[alias];
}

std::string ASTDriver::lookup_var_name(const std::string alias)
{
std::tuple<std::string, indx_type> var = aliases[alias];
return std::get<0>(var);
}

indx_type ASTDriver::lookup_var_indices(const std::string alias)
{
std::tuple<std::string, indx_type> var = aliases[alias];
return std::get<1>(var);
}

void ASTDriver::add_lookup_entry(std::string alias, std::string var_name, indx_type indices)
{
aliases.insert({alias, {var_name, indices}});
}

void ASTDriver::add_lookup_entry(std::string alias, std::string var_name)
{
aliases.insert({alias, {var_name, {}}});
}

void ASTDriver::createNode(std::string op_name, size_t numsubexprs)
{
ASTNode *node = new ASTNode(op_name, numsubexprs);
for (size_t i = 1; i <= numsubexprs; ++i)
{
// TODO: check that holding contains ASTNode(s)
// else throw error that parsing failed
ASTNode *subexpr = holding.top();
node->insert_subexpr_n(subexpr, numsubexprs - i);
holding.pop();
}
holding.push(node);
}

void ASTDriver::createNode(std::string alias)
{
ASTNode *node = new ASTNode("ALIAS", alias);
holding.push(node);
}

void ASTDriver::createNode(std::string alias, indx_type indices)
{
ASTNode *node = new ASTNode("INDEX", indices);
node->pushback_subexpr(new ASTNode("ALIAS", alias));
holding.push(node);
}

}
}
Loading
Loading