Reading from Parquet file generates an assertion error

[metasim]

I'm guessing something changed between Spark 2.2 and 2.3 that we didn't catch because the unit tests around Parquet IO in EncodingSpec only counted rows, and didn't perform any action that realized the tiles.

java.lang.AssertionError: assertion failed: User-defined types in Catalyst schema should have already been expanded:

{
  "type" : "struct",
  "fields" : [ {
    "name" : "tile",
    "type" : {
      "type" : "struct",
      "fields" : [ {
        "name" : "cell_context",
        "type" : {
          "type" : "struct",
          "fields" : [ {
            "name" : "cellType",
            "type" : {
              "type" : "struct",
              "fields" : [ {
                "name" : "cellTypeName",
                "type" : "string",
                "nullable" : false,
                "metadata" : { }
              } ]
            },
            "nullable" : false,
            "metadata" : { }
          }, {
            "name" : "dimensions",
            "type" : {
              "type" : "struct",
              "fields" : [ {
                "name" : "cols",
                "type" : "short",
                "nullable" : false,
                "metadata" : { }
              }, {
                "name" : "rows",
                "type" : "short",
                "nullable" : false,
                "metadata" : { }
              } ]
            },
            "nullable" : false,
            "metadata" : { }
          } ]
        },
        "nullable" : false,
        "metadata" : { }
      }, {
        "name" : "cell_data",
        "type" : {
          "type" : "struct",
          "fields" : [ {
            "name" : "cells",
            "type" : "binary",
            "nullable" : true,
            "metadata" : { }
          }, {
            "name" : "ref",
            "type" : {
              "type" : "struct",
              "fields" : [ {
                "name" : "source",
                "type" : {
                  "type" : "udt",
                  "class" : "org.apache.spark.sql.rf.RasterSourceUDT",
                  "pyClass" : "pyrasterframes.rf_types.RasterSourceUDT",
                  "sqlType" : {
                    "type" : "struct",
                    "fields" : [ {
                      "name" : "raster_source_kryo",
                      "type" : "binary",
                      "nullable" : false,
                      "metadata" : { }
                    } ]
                  }
                },
                "nullable" : false,
                "metadata" : { }
              }, {
                "name" : "bandIndex",
                "type" : "integer",
                "nullable" : false,
                "metadata" : { }
              }, {
                "name" : "subextent",
                "type" : {
                  "type" : "struct",
                  "fields" : [ {
                    "name" : "xmin",
                    "type" : "double",
                    "nullable" : false,
                    "metadata" : { }
                  }, {
                    "name" : "ymin",
                    "type" : "double",
                    "nullable" : false,
                    "metadata" : { }
                  }, {
                    "name" : "xmax",
                    "type" : "double",
                    "nullable" : false,
                    "metadata" : { }
                  }, {
                    "name" : "ymax",
                    "type" : "double",
                    "nullable" : false,
                    "metadata" : { }
                  } ]
                },
                "nullable" : true,
                "metadata" : { }
              }, {
                "name" : "subgrid",
                "type" : {
                  "type" : "struct",
                  "fields" : [ {
                    "name" : "colMin",
                    "type" : "integer",
                    "nullable" : false,
                    "metadata" : { }
                  }, {
                    "name" : "rowMin",
                    "type" : "integer",
                    "nullable" : false,
                    "metadata" : { }
                  }, {
                    "name" : "colMax",
                    "type" : "integer",
                    "nullable" : false,
                    "metadata" : { }
                  }, {
                    "name" : "rowMax",
                    "type" : "integer",
                    "nullable" : false,
                    "metadata" : { }
                  } ]
                },
                "nullable" : true,
                "metadata" : { }
              } ]
            },
            "nullable" : true,
            "metadata" : { }
          } ]
        },
        "nullable" : false,
        "metadata" : { }
      } ]
    },
    "nullable" : true,
    "metadata" : { }
  } ]
}

[metasim]

Problem is likely having a UDT inside a UDT

                  "class" : "org.apache.spark.sql.rf.RasterSourceUDT",