From f15f947c09669461fab1496ff334ee2a3da79f5b Mon Sep 17 00:00:00 2001 From: Brian Silver Date: Tue, 16 Aug 2016 14:39:25 -0500 Subject: [PATCH] Update memory library for 1R 4gbx4 DIMM Add SPD to ekb, including Hynix power-on DIMM and 4R VBU DIMM Update SPD blobs to match recent changes to VBU SPD Change-Id: I386c55a0c86a0c6d21fcaa2451830ed44825aa95 Original-Change-Id: I1ea6be55858a3bd8ec206624294a3a2accd81136 Reviewed-on: http://ralgit01.raleigh.ibm.com/gerrit1/28349 Tested-by: Jenkins Server Reviewed-by: Louis Stermole Reviewed-by: STEPHEN GLANCY Tested-by: Hostboot CI Reviewed-by: Christian R. Geddes Reviewed-by: Jennifer A. Stofer Reviewed-on: http://ralgit01.raleigh.ibm.com/gerrit1/37403 Reviewed-by: Daniel M. Crowell Tested-by: Daniel M. Crowell --- .../p9/procedures/hwp/memory/lib/mc/xlate.C | 388 +++++++++++++----- .../p9/procedures/hwp/memory/lib/mc/xlate.H | 96 +++++ 2 files changed, 372 insertions(+), 112 deletions(-) diff --git a/src/import/chips/p9/procedures/hwp/memory/lib/mc/xlate.C b/src/import/chips/p9/procedures/hwp/memory/lib/mc/xlate.C index 007890eb5b4..432015dd9db 100644 --- a/src/import/chips/p9/procedures/hwp/memory/lib/mc/xlate.C +++ b/src/import/chips/p9/procedures/hwp/memory/lib/mc/xlate.C @@ -41,174 +41,337 @@ #include #include +#include #include +#include #include -using fapi2::TARGET_TYPE_MCBIST; -using fapi2::TARGET_TYPE_PROC_CHIP; -using fapi2::TARGET_TYPE_SYSTEM; using fapi2::TARGET_TYPE_MCA; -using fapi2::TARGET_TYPE_MCS; using fapi2::TARGET_TYPE_DIMM; -using fapi2::FAPI2_RC_SUCCESS; - namespace mss { namespace mc { +/// A little vector of translators. We have one of these for each DIMM we support +static const std::vector xlate_map = +{ + // 2R4Gbx4 DDR4 RDIMM + { + dimm::kind(2, 0, 4, 4, fapi2::ENUM_ATTR_EFF_DRAM_GEN_DDR4, fapi2::ENUM_ATTR_EFF_DIMM_TYPE_RDIMM, 16, 16), + xlate_dimm_2R4Gbx4 + }, + { + dimm::kind(1, 0, 4, 4, fapi2::ENUM_ATTR_EFF_DRAM_GEN_DDR4, fapi2::ENUM_ATTR_EFF_DIMM_TYPE_RDIMM, 16, 8), + xlate_dimm_1R4Gbx4 + }, + +}; + /// -/// @brief Perform initializations of the MC translation -/// @tparm P the fapi2::TargetType of the port -/// @tparm TT the typename of the traits -/// @param[in] i_target, the target which has the MCA to map -/// @return FAPI2_RC_SUCCESS iff ok +/// @brief Helper to lay down the col and bank mappings. +/// @param[in] o_xlate1 a buffer representing the xlate register to modify +/// @param[in] o_xlate2 a buffer representing the xlate register to modify +/// @note This is for 16 bank DIMM, 32 bank DIMM will be different /// -template<> -fapi2::ReturnCode setup_xlate_map(const fapi2::Target& i_target) +static void column_and_16bank_helper(fapi2::buffer& l_xlate1, fapi2::buffer& l_xlate2) { - fapi2::buffer l_xlate; - fapi2::buffer l_xlate1; - fapi2::buffer l_xlate2; + // These are compile time freebies, so there's no need to bother putting them in a pre-defined + // constant and or-ing them in. Keeps things much more clear when the performance team wants to muck + // around with the settings. Mappings taken directly from the Nimbus Workbook. The magic numbers + // aren't; they're settings as defined in the scomdef - const auto l_dimms = i_target.getChildren(); + l_xlate1.insertFromRight(0b01101); - FAPI_INF("Setting up xlate registers for MCA%d (%d)", mss::pos(i_target), mss::index(i_target)); + l_xlate1.insertFromRight(0b01100); - // We enable the DIMM select bit for slot1 if we have two DIMM installed - l_xlate.writeBit(l_dimms.size() == 2); + l_xlate1.insertFromRight(0b01011); - // Get the functional DIMM on this port. - for (auto d : l_dimms) - { - // Our slot (0, 1) is the same as our general index. - const uint64_t l_slot = mss::index(d); + l_xlate1.insertFromRight(0b01010); - // Our slot offset tells us which 16 bit section in the xlt register to use for this DIMM - // We'll either use the left most bits (slot 0) or move 16 bits to the right for slot 1. - const uint64_t l_slot_offset = l_slot * 16; + l_xlate2.insertFromRight(0b01001); + + l_xlate2.insertFromRight(0b00111); + + l_xlate2.insertFromRight(0b01110); - // Get the translation array, based on this specific DIMM's config - dimm::kind l_dimm(d); + l_xlate2.insertFromRight(0b10000); - FAPI_DBG("address translation for DIMM %s %dR %dgbx%d in slot %d", - mss::c_str(d), l_dimm.iv_master_ranks, l_dimm.iv_dram_density, l_dimm.iv_dram_width, l_slot); + l_xlate2.insertFromRight(0b10001); + l_xlate2.insertFromRight(0b10010); +} - // Set the proper bit if there is a DIMM in this slot. If there wasn't, we wouldn't see - // this DIMM in the vector, so this is always safe. - l_xlate.setBit(MCS_PORT02_MCP0XLT0_SLOT0_VALID + l_slot_offset); +/// +/// @brief Perform initializations of the MC translation +/// @param[in] i_kind the DIMM to map +/// @param[in] i_offset the offset; whether the DIMM ins slot 0 or slot 1 +/// @param[in] i_largest whether or not we're the largest DIMM on the port. +/// @param[in] o_xlate a buffer representing the xlate register to modify +/// @param[in] o_xlate1 a buffer representing the xlate register to modify +/// @param[in] o_xlate2 a buffer representing the xlate register to modify +/// @note Called for 2R4Gbx4 DDR4 RDIMM +/// +void xlate_dimm_2R4Gbx4( const dimm::kind& i_kind, + const uint64_t i_offset, + const bool i_largest, + fapi2::buffer& o_xlate, + fapi2::buffer& o_xlate1, + fapi2::buffer& o_xlate2 ) +{ + // Set the proper bit if there is a DIMM in this slot. If there wasn't, we wouldn't see + // this DIMM in the vector, so this is always safe. + o_xlate.setBit(MCS_PORT02_MCP0XLT0_SLOT0_VALID + i_offset); + + // Check our master ranks, and enable the proper bits. + // Note this seems a little backward. M0 is the left most bit, M1 the right most. + // So, M1 changes for ranks 0,1 and M0 changes for ranks 3,4 + // 2 rank DIMM, so master bit 1 (least significant) bit needs to be mapped. + o_xlate.setBit(MCS_PORT13_MCP0XLT0_SLOT0_M1_VALID + i_offset); + o_xlate.insertFromRight(0b01111); + + // Tell the MC which of the row bits are valid, and map the DIMM selector + // We're a 16 row DIMM, so ROW15 is valid. + o_xlate.setBit(MCS_PORT02_MCP0XLT0_SLOT0_ROW15_VALID + i_offset); + o_xlate.insertFromRight(0b00110); + // Drop down the column assignments + column_and_16bank_helper(o_xlate1, o_xlate2); - // Set the 12G DIMM bit if either DIMM is 12G - // Is this correct? BRS - l_xlate.writeBit(l_dimm.iv_size == 12); + // Setup the D-bit. If we're the largest DIMM, it is our mapping which maters. + // Notice tht we don't care if the D-value bit has been set; this mapping needs to be setup regardless (SJ Powell says so) + if (i_largest) + { + FAPI_INF("setting d-bit mapping (am largest) for %s", mss::c_str(i_kind.iv_target)); + o_xlate.insertFromRight(0b00101, MCS_PORT02_MCP0XLT0_D_BIT_MAP + i_offset, MCS_PORT02_MCP0XLT0_D_BIT_MAP_LEN); + } +} - // Check our master ranks, and enable the proper bits. - // Note this seems a little backward. M0 is the left most bit, M1 the right most. - // So, M1 changes for ranks 0,1 and M0 changes for ranks 3,4 - if (l_dimm.iv_master_ranks > 0) - { - l_xlate.setBit(MCS_PORT13_MCP0XLT0_SLOT0_M1_VALID + l_slot_offset); - } - if (l_dimm.iv_master_ranks > 2) - { - l_xlate.setBit(MCS_PORT13_MCP0XLT0_SLOT0_M0_VALID + l_slot_offset); - } +/// +/// @brief Perform initializations of the MC translation +/// @param[in] i_kind the DIMM to map +/// @param[in] i_offset the offset; whether the DIMM ins slot 0 or slot 1 +/// @param[in] i_largest whether or not we're the largest DIMM on the port. +/// @param[in] o_xlate a buffer representing the xlate register to modify +/// @param[in] o_xlate1 a buffer representing the xlate register to modify +/// @param[in] o_xlate2 a buffer representing the xlate register to modify +/// @note Called for 1R4Gbx4 DDR4 RDIMM +/// +void xlate_dimm_1R4Gbx4( const dimm::kind& i_kind, + const uint64_t i_offset, + const bool i_largest, + fapi2::buffer& o_xlate, + fapi2::buffer& o_xlate1, + fapi2::buffer& o_xlate2 ) +{ + // 1R DIMM are special. We need to handle 2 1R DIMM on a port as a special case - kind of make them look like + // a single 2R DIMM. So we have to do a little dance here to get our partners configuration. + const auto& l_mca = mss::find_target(i_kind.iv_target); + const auto& l_dimms = mss::find_targets(l_mca); + const std::vector l_dimm_kinds = dimm::kind::vector(l_dimms); + bool l_all_slots_1R = false; - // Check slave ranks - // Note this sems a little backward. S0 is the left-most slave bit. So, - // if there are more than 0 slave ranks, S2 will increment first. - if (l_dimm.iv_slave_ranks > 0) - { - l_xlate.setBit(MCS_PORT13_MCP0XLT0_SLOT0_S2_VALID + l_slot_offset); - } + // If we only have 1 DIMM, we don't have two slots with 1R DIMM. If we need to check, iterate + // over the DIMM kinds and make sure all the DIMM have one master and zero slaves. + if (l_dimms.size() > 1) + { + l_all_slots_1R = true; - if (l_dimm.iv_slave_ranks > 2) + for (const auto& k : l_dimm_kinds) { - l_xlate.setBit(MCS_PORT13_MCP0XLT0_SLOT0_S1_VALID + l_slot_offset); + l_all_slots_1R &= (k.iv_master_ranks == 1) && (k.iv_slave_ranks == 0); } - if (l_dimm.iv_slave_ranks > 4) - { - l_xlate.setBit(MCS_PORT13_MCP0XLT0_SLOT0_S0_VALID + l_slot_offset); - } + FAPI_INF("We have a 1R DIMM and more than one DIMM installed; all 1R? %s", + (l_all_slots_1R == true ? "yes" : "no") ); + } + // Set the proper bit if there is a DIMM in this slot. If there wasn't, we wouldn't see + // this DIMM in the vector, so this is always safe. + o_xlate.setBit(MCS_PORT02_MCP0XLT0_SLOT0_VALID + i_offset); + // If we have all the slots filled in with 1R/0 slave DIMM, we build a very differnt mapping. + if (l_all_slots_1R) + { // Tell the MC which of the row bits are valid, and map the DIMM selector - if (l_dimm.iv_rows >= 16) - { - l_xlate.setBit(MCS_PORT02_MCP0XLT0_SLOT0_ROW15_VALID + l_slot_offset); - l_xlate.insertFromRight(0b00110); - l_xlate.insertFromRight(0b00101); - } + // We're a 16 row DIMM, so ROW15 is valid. + o_xlate.setBit(MCS_PORT02_MCP0XLT0_SLOT0_ROW15_VALID + i_offset); + o_xlate.insertFromRight(0b00110); - if (l_dimm.iv_rows >= 17) - { - l_xlate.setBit(MCS_PORT02_MCP0XLT0_SLOT0_ROW16_VALID + l_slot_offset); - l_xlate.insertFromRight(0b00101); - l_xlate.insertFromRight(0b00100); - } + // Drop down the column assignments. + column_and_16bank_helper(o_xlate1, o_xlate2); - if (l_dimm.iv_rows >= 18) - { - l_xlate.setBit(MCS_PORT02_MCP0XLT0_SLOT0_ROW17_VALID + l_slot_offset); - l_xlate.insertFromRight(0b00100); - l_xlate.insertFromRight(0b00011); - } + // Setup the D-bit. Since both DIMM are identical, we just need to setup the map + FAPI_INF("setting d-bit mapping (all 1R DIMM) for %s", mss::c_str(i_kind.iv_target)); + o_xlate.insertFromRight(0b01111); + return; } - // TK: remove and make general in the loop above BRS + // So if we're here we have only 1 1R DIMM installed. This translation is different. - // Two rank DIMM, so master bit 1 (least significant) bit needs to be mapped. - l_xlate.insertFromRight(0b01111); + // Tell the MC which of the row bits are valid, and map the DIMM selector + // We're a 16 row DIMM, so ROW15 is valid. + o_xlate.setBit(MCS_PORT02_MCP0XLT0_SLOT0_ROW15_VALID + i_offset); + o_xlate.insertFromRight(0b00111); - // Slot 1 isn't populated, so forget those bits for now. + // We don't just drop down the col and bank assignments, they're different. + o_xlate1.insertFromRight(0b01110); - // DIMM bit map isn't exactly ignored for only one populated slot. It still needs to be - // set in the map. Per S. Powell. - // Master rank 0, 1 bit maps are ignored. - // Row 16,17 bit maps are ignored. - // Row 15 maps to Port Address bit 6 + o_xlate1.insertFromRight(0b01101); - // Drop down the column assignments - l_xlate1.insertFromRight(0b01101); + o_xlate1.insertFromRight(0b01100); - l_xlate1.insertFromRight(0b01100); + o_xlate1.insertFromRight(0b01011); - l_xlate1.insertFromRight(0b01011); + o_xlate2.insertFromRight(0b01010); - l_xlate1.insertFromRight(0b01010); + o_xlate2.insertFromRight(0b01001); - l_xlate2.insertFromRight(0b01001); + o_xlate2.insertFromRight(0b01111); - l_xlate2.insertFromRight(0b00111); - - l_xlate2.insertFromRight(0b01110); - - l_xlate2.insertFromRight(0b10000); - l_xlate2.insertFromRight(0b10001); - l_xlate2.insertFromRight(0b10010); - FAPI_DBG("HACK: Cramming 0x%016lx in for MCP0XLT0", l_xlate); - FAPI_DBG("HACK: Cramming 0x%016lx in for MCP0XLT1", l_xlate1); - FAPI_DBG("HACK: Cramming 0x%016lx in for MCP0XLT2", l_xlate2); + // There's nothing to do for the D-bit. We're either not the largest DIMM, in which case the lrgest DIMM + // will fix up our D-bit mapping, or we're the only DIMM in the port. If we're the only DIMM in the port, + // there is no D-bit mapping for a 1 slot 1R DIMM. + return; +} + +/// +/// @brief Perform initializations of the MC translation - MCA specialization +/// @param[in] i_target, the target which has the MCA to map +/// @return FAPI2_RC_SUCCESS iff ok +/// +template<> +fapi2::ReturnCode setup_xlate_map(const fapi2::Target& i_target) +{ + fapi2::buffer l_xlate; + fapi2::buffer l_xlate1; + fapi2::buffer l_xlate2; + + const auto l_dimms = i_target.getChildren(); + + // We need to keep around specifications of both DIMM as we set the D bit based on the sizes of the DIMM + std::vector l_dimm_kinds = dimm::kind::vector(l_dimms); + + FAPI_INF("Setting up xlate registers for MCA%d (%d)", mss::pos(i_target), mss::index(i_target)); + + // Considering the DIMM, record who gets the D bit. We make sure the *smallest* DIMM has the highest address + // range by setting it's D bit to 1. This eliminates, or reduces, holes in the memory map. + // However, we need to set that DIMM's D bit in the location of the largest DIMM's D-bit map (I know that's + // hard to grok - set the D bit in the smallest DIMM but in the location mapped for the largest.) So we + // keep track of the largest DIMM so when we set it up, we make sure to set the D-bit in the other. + std::sort(l_dimm_kinds.begin(), l_dimm_kinds.end(), [](const dimm::kind & a, const dimm::kind & b) -> bool + { + return a.iv_size > b.iv_size; + }); + + FAPI_INF("DIMM with the largest size on this port is %s %dR %dgbx%d (%dG)", + mss::c_str(l_dimm_kinds[0].iv_target), + l_dimm_kinds[0].iv_master_ranks, l_dimm_kinds[0].iv_dram_density, + l_dimm_kinds[0].iv_dram_width, l_dimm_kinds[0].iv_size); + + const auto l_d_bit_target = l_dimm_kinds[0].iv_target; + + // Get the functional DIMM on this port. + for (const auto& d : l_dimms) + { + // Our slot (0, 1) is the same as our general index. + const uint64_t l_slot = mss::index(d); + + // Our slot offset tells us which 16 bit section in the xlt register to use for this DIMM + // We'll either use the left most bits (slot 0) or move 16 bits to the right for slot 1. + const uint64_t l_slot_offset = l_slot * 16; + + // Grab our kind out of the vector + auto l_kind = std::find_if(l_dimm_kinds.begin(), l_dimm_kinds.end(), [&l_slot](const dimm::kind & k) -> bool + { + return mss::index(k.iv_target) == l_slot; + }); + + // If we don't find the fellow, we have a programming bug as we made the kind vector from the + // vector we're iterating over. + if (l_kind == l_dimm_kinds.end()) + { + FAPI_ERR("can't find our dimm in the kind vector: l_slot %d %s", l_slot, mss::c_str(d)); + fapi2::Assert(false); + } + + FAPI_DBG("address translation for DIMM %s %dR %dgbx%d (%dG) in slot %d", + mss::c_str(d), l_kind->iv_master_ranks, l_kind->iv_dram_density, + l_kind->iv_dram_width, l_kind->iv_size, l_slot); + + // Set the proper bit if there is a DIMM in this slot. If there wasn't, we wouldn't see + // this DIMM in the vector, so this is always safe. + l_xlate.setBit(MCS_PORT02_MCP0XLT0_SLOT0_VALID + l_slot_offset); + + // Find the proper set function based on this DIMM kind. + const auto l_setup = std::find_if( xlate_map.begin(), xlate_map.end(), [l_kind](const xlate_setup & x) -> bool + { + return x.iv_kind == *l_kind; + } ); + + // If we're the smallest DIMM in the port and we have more than one DIMM, we set our D-bit. + if( (l_d_bit_target != d) && (l_dimms.size() > 1) ) + { + FAPI_INF("noting d-bit of 1 for %s", mss::c_str(d)); + l_xlate.setBit(MCS_PORT13_MCP0XLT0_SLOT0_D_VALUE + l_slot_offset); + } + + // If we didn't find it, raise a stink. + FAPI_ASSERT( l_setup != xlate_map.end(), + fapi2::MSS_NO_XLATE_FOR_DIMM(). + set_DIMM_IN_ERROR(d). + set_MASTER_RANKS(l_kind->iv_master_ranks). + set_SLAVE_RANKS(l_kind->iv_slave_ranks). + set_DRAM_DENSITY(l_kind->iv_dram_density). + set_DRAM_WIDTH(l_kind->iv_dram_width). + set_DRAM_GENERATION(l_kind->iv_dram_generation). + set_DIMM_TYPE(l_kind->iv_dimm_type). + set_ROWS(l_kind->iv_rows). + set_SIZE(l_kind->iv_size), + "no address translation funtion for DIMM %s %dR %dgbx%d (%dG) in slot %d", + mss::c_str(d), l_kind->iv_master_ranks, l_kind->iv_dram_density, + l_kind->iv_dram_width, l_kind->iv_size, l_slot ); + + // If we did find it, call the translation function to fill in the blanks. + // The conditional argument tells the setup function whether this setup sould set the D bit, as we're + // the largest DIMM on the port. + l_setup->iv_func(*l_kind, l_slot_offset, (l_kind->iv_target == l_d_bit_target), l_xlate, l_xlate1, l_xlate2); + } + + + FAPI_INF("cramming 0x%016lx in for MCP0XLT0", l_xlate); + FAPI_INF("cramming 0x%016lx in for MCP0XLT1", l_xlate1); + FAPI_INF("cramming 0x%016lx in for MCP0XLT2", l_xlate2); FAPI_TRY( mss::putScom(i_target, MCA_MBA_MCP0XLT0, l_xlate) ); FAPI_TRY( mss::putScom(i_target, MCA_MBA_MCP0XLT1, l_xlate1) ); @@ -217,5 +380,6 @@ fapi2::ReturnCode setup_xlate_map(const fapi2::Target& i_target fapi_try_exit: return fapi2::current_err; } + } // namespace mc } // namespace mss diff --git a/src/import/chips/p9/procedures/hwp/memory/lib/mc/xlate.H b/src/import/chips/p9/procedures/hwp/memory/lib/mc/xlate.H index b4bee87d85c..850ed9be7e6 100644 --- a/src/import/chips/p9/procedures/hwp/memory/lib/mc/xlate.H +++ b/src/import/chips/p9/procedures/hwp/memory/lib/mc/xlate.H @@ -22,3 +22,99 @@ /* permissions and limitations under the License. */ /* */ /* IBM_PROLOG_END_TAG */ + +/// +/// @file xmalte.H +/// @brief Definitions for translation registers +/// +// *HWP HWP Owner: Brian Silver +// *HWP HWP Backup: Andre Marin +// *HWP Team: Memory +// *HWP Level: 2 +// *HWP Consumed by: HB:FSP + +#ifndef _MSS_XLT_H_ +#define _MSS_XLT_H_ + +#include + +#include +#include + +#include +#include +#include +#include + +namespace mss +{ + +namespace mc +{ + +/// +/// @brief A small class to represent the setup of a translation register based on DIMM characteristics +/// +struct xlate_setup +{ + /// + /// @brief Constructor to make a translation register setup structure. + /// @param[in] i_kind a DIMM kind structure representing the ... err... kind of DIMM + /// @param[in] i_func a function pointer to a function which does the configuring + /// + xlate_setup( const dimm::kind i_kind, + void (*i_func)( const dimm::kind&, const uint64_t, const bool, + fapi2::buffer&, fapi2::buffer&, fapi2::buffer& ) ): + iv_kind(i_kind), + iv_func(i_func) + { + } + + // Keep around the kind of DIMM this nugget represents + dimm::kind iv_kind; + + // The function to call to setup the translation registers to setup for our DIMM kind. + void (*iv_func)( const dimm::kind&, const uint64_t, const bool, + fapi2::buffer&, fapi2::buffer&, fapi2::buffer& ); +}; + + +/// +/// @brief Perform initializations of the MC translation +/// @param[in] i_kind the DIMM to map +/// @param[in] i_offset the offset; whether the DIMM ins slot 0 or slot 1 +/// @param[in] i_largest whether or not we're the largest DIMM on the port. +/// @param[in] o_xlate a buffer representing the xlate register to modify +/// @param[in] o_xlate1 a buffer representing the xlate register to modify +/// @param[in] o_xlate2 a buffer representing the xlate register to modify +/// @note Called for 2R4Gbx4 DDR4 RDIMM +/// +void xlate_dimm_2R4Gbx4( const dimm::kind& i_kind, + const uint64_t i_offset, + const bool i_largest, + fapi2::buffer& o_xlate, + fapi2::buffer& o_xlate1, + fapi2::buffer& o_xlate2 ); + +/// +/// @brief Perform initializations of the MC translation +/// @param[in] i_kind the DIMM to map +/// @param[in] i_offset the offset; whether the DIMM ins slot 0 or slot 1 +/// @param[in] i_largest whether or not we're the largest DIMM on the port. +/// @param[in] o_xlate a buffer representing the xlate register to modify +/// @param[in] o_xlate1 a buffer representing the xlate register to modify +/// @param[in] o_xlate2 a buffer representing the xlate register to modify +/// @note Called for 1R4Gbx4 DDR4 RDIMM +/// +void xlate_dimm_1R4Gbx4( const dimm::kind& i_kind, + const uint64_t i_offset, + const bool i_largest, + fapi2::buffer& o_xlate, + fapi2::buffer& o_xlate1, + fapi2::buffer& o_xlate2 ); + +} // ns mc + +} // ns mss + +#endif