xxxxxxxxxx
def test_my_case(spark_context):
test_rdd = spark_context.parallelize([1, 2, 3, 4])
# ...
xxxxxxxxxx
{
'spark.app.name': 'pytest-spark',
'spark.default.parallelism': 1,
'spark.dynamicAllocation.enabled': 'false',
'spark.executor.cores': 1,
'spark.executor.instances': 1,
'spark.io.compression.codec': 'lz4',
'spark.rdd.compress': 'false',
'spark.sql.shuffle.partitions': 1,
'spark.shuffle.compress': 'false',
'spark.sql.catalogImplementation': 'hive',
}
xxxxxxxxxx
def test_spark_session_dataframe(spark_session):
test_df = spark_session.createDataFrame([[1,3],[2,4]], "a: int, b: int")
# ...
xxxxxxxxxx
[pytest]
spark_home = /opt/spark
spark_options =
spark.app.name: my-pytest-spark-tests
spark.executor.instances: 1
spark.jars.packages: com.databricks:spark-xml_2.12:0.5.0