diff --git a/connector/avro/src/main/scala/org/apache/spark/sql/v2/avro/AvroTable.scala b/connector/avro/src/main/scala/org/apache/spark/sql/v2/avro/AvroTable.scala index e898253be1168..abcea9a2a238e 100644 --- a/connector/avro/src/main/scala/org/apache/spark/sql/v2/avro/AvroTable.scala +++ b/connector/avro/src/main/scala/org/apache/spark/sql/v2/avro/AvroTable.scala @@ -51,5 +51,5 @@ case class AvroTable( override def supportsDataType(dataType: DataType): Boolean = AvroUtils.supportsDataType(dataType) - override def formatName: String = "AVRO" + override def formatName: String = "Avro" } diff --git a/connector/avro/src/test/scala/org/apache/spark/sql/avro/AvroSuite.scala b/connector/avro/src/test/scala/org/apache/spark/sql/avro/AvroSuite.scala index c55113ec3ef73..fac19a3b8bcf3 100644 --- a/connector/avro/src/test/scala/org/apache/spark/sql/avro/AvroSuite.scala +++ b/connector/avro/src/test/scala/org/apache/spark/sql/avro/AvroSuite.scala @@ -22,7 +22,7 @@ import java.net.URI import java.nio.file.{Files, Paths, StandardCopyOption} import java.sql.{Date, Timestamp} import java.time.{LocalDate, LocalDateTime} -import java.util.UUID +import java.util.{Collections => JCollections, UUID} import scala.jdk.CollectionConverters._ @@ -44,7 +44,7 @@ import org.apache.spark.sql.catalyst.util.DateTimeTestUtils import org.apache.spark.sql.catalyst.util.DateTimeTestUtils.{withDefaultTimeZone, LA, UTC} import org.apache.spark.sql.execution.{FormattedMode, SparkPlan} import org.apache.spark.sql.execution.datasources.{CommonFileDataSourceSuite, DataSource, FilePartition} -import org.apache.spark.sql.execution.datasources.v2.BatchScanExec +import org.apache.spark.sql.execution.datasources.v2.{BatchScanExec, FileDataSourceV2, FileTable} import org.apache.spark.sql.functions._ import org.apache.spark.sql.internal.LegacyBehaviorPolicy import org.apache.spark.sql.internal.LegacyBehaviorPolicy._ @@ -3483,6 +3483,18 @@ class AvroV2Suite extends AvroSuite with ExplainSuiteHelper { } } + test("SPARK-56457: Avro V2 formatName matches V1 FileFormat.toString") { + val v2Provider = DataSource.lookupDataSourceV2("avro", spark.sessionState.conf) + assert(v2Provider.isDefined) + val dsV2 = v2Provider.get.asInstanceOf[FileDataSourceV2] + val v1Format = dsV2.fallbackFileFormat.getDeclaredConstructor().newInstance() + val emptyProps = JCollections.emptyMap[String, String]() + val v2Table = dsV2.getTable( + new StructType(), Array.empty, emptyProps).asInstanceOf[FileTable] + assert(v2Table.formatName == v1Format.toString, + s"V2 formatName '${v2Table.formatName}' != V1 toString '${v1Format.toString}'") + } + test("Geospatial types are not supported in Avro") { withTempDir { dir => // Temporary directory for writing the test data. diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/v2/FileTableSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/v2/FileTableSuite.scala index 0d18e3bf809e0..0269bb464c872 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/v2/FileTableSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/v2/FileTableSuite.scala @@ -16,6 +16,8 @@ */ package org.apache.spark.sql.execution.datasources.v2 +import java.util.{Collections => JCollections} + import scala.jdk.CollectionConverters._ import org.apache.hadoop.fs.FileStatus @@ -95,6 +97,23 @@ class FileTableSuite extends QueryTest with SharedSparkSession { } } + test("SPARK-56457: V2 FileTable.formatName matches V1 FileFormat.toString") { + withSQLConf(SQLConf.USE_V1_SOURCE_LIST.key -> "") { + allFileBasedDataSources.foreach { format => + val v2Provider = DataSource.lookupDataSourceV2(format, spark.sessionState.conf) + assert(v2Provider.isDefined, s"Expected V2 provider for $format") + val dsV2 = v2Provider.get.asInstanceOf[FileDataSourceV2] + val v1Format = dsV2.fallbackFileFormat.getDeclaredConstructor().newInstance() + val emptyProps = JCollections.emptyMap[String, String]() + val v2Table = dsV2.getTable( + new StructType(), Array.empty, emptyProps).asInstanceOf[FileTable] + assert(v2Table.formatName == v1Format.toString, + s"V2 formatName '${v2Table.formatName}' != " + + s"V1 toString '${v1Format.toString}' for format '$format'") + } + } + } + allFileBasedDataSources.foreach { format => test("SPARK-49519, SPARK-50287: Merge options of table and relation when " + s"constructing ScanBuilder and WriteBuilder in FileFormat - $format") {