PySpark Cheatsheet
Setup
from pyspark.sql import SparkSession
spark = (
SparkSession.builder
.appName("MyApp")
.config("spark.sql.shuffle.partitions", "200")
.config("spark.executor.memory", "4g")
.getOrCreate()
)
spark.sparkContext.setLogLevel("WARN")
spark.version
spark.stop()
Creating DataFrames
from pyspark.sql import Row
import pyspark.sql.types as T
df = spark.createDataFrame([(1, "a"), (2, "b")], ["id", "name"])
schema = T.StructType([
T.StructField("id", T.IntegerType(), nullable=False),
T.StructField("name", T.StringType(), nullable=True),
])
df = spark.createDataFrame(data, schema=schema)
df = spark.read.csv("path.csv", header=True, inferSchema=True)
df = spark.read.parquet("path.parquet")
df = spark.read.json("path.json")
df = spark.read.format("jdbc").option("url", jdbc_url).option("dbtable", "table").load()
df = spark.table("catalog.schema.table") # Delta/managed table
df = spark.sql("SELECT * FROM some_table")
df.write.mode("overwrite").parquet("out.parquet")
df.write.mode("append").saveAsTable("catalog.schema.table")
df.write.partitionBy("year", "month").parquet("out/")
df.write.format("delta").mode("overwrite").save("path/")
Inspecting Data
df.show() # default 20 rows
df.show(5, truncate=False)
df.printSchema()
df.columns
df.dtypes
df.count()
df.describe().show()
df.summary().show() # more detailed stats incl. percentiles
df.explain() # physical/logical plan
df.explain(mode="formatted") # readable query plan
df.rdd.getNumPartitions()
Selecting & Filtering
from pyspark.sql import functions as F
df.select("a", "b")
df.select(F.col("a"), F.col("b").alias("beta"))
df.select("*", (F.col("a") + F.col("b")).alias("c"))
df.filter(F.col("a") > 5)
df.where(F.col("a") > 5) # alias for filter
df.filter((F.col("a") > 5) & (F.col("b") < 10))
df.filter(F.col("cat").isin(["x", "y"]))
df.filter(F.col("name").rlike("^prefix.*"))
df.filter(F.col("a").isNull())
df.filter(F.col("a").isNotNull())
df.distinct()
df.dropDuplicates(["a"])
df.limit(10)
Modifying Data
df.withColumn("c", F.col("a") + F.col("b"))
df.withColumn("flag", F.when(F.col("a") > 5, "high").otherwise("low"))
df.withColumnRenamed("a", "alpha")
df.withColumns({"c": F.col("a") + 1, "d": F.col("b") * 2}) # multiple at once (Spark 3.3+)
df.drop("a")
df.na.drop() # drop rows with any null
df.na.drop(subset=["a"])
df.na.fill(0)
df.na.fill({"a": 0, "b": "unknown"})
df.na.replace(["old"], ["new"], "col")
df = df.withColumn("a", F.col("a").cast(T.IntegerType()))
df = df.withColumn("date", F.to_date("date_str", "yyyy-MM-dd"))
df.sort("a")
df.sort(F.col("a").desc())
df.orderBy(F.col("a").desc(), F.col("b").asc())
Aggregation & GroupBy
df.groupBy("category").sum("value")
df.groupBy("category").agg(
F.sum("value").alias("total"),
F.avg("value").alias("avg"),
F.count("*").alias("n"),
F.max("value").alias("max_val"),
F.countDistinct("id").alias("unique_ids"),
)
df.groupBy("cat1", "cat2").agg(F.sum("value"))
df.groupBy("category").pivot("year").agg(F.sum("value")) # pivot table
df.agg(F.sum("value"), F.avg("value")) # no grouping — whole df
# Window functions
from pyspark.sql.window import Window
w = Window.partitionBy("category").orderBy("date")
df.withColumn("running_total", F.sum("value").over(w))
df.withColumn("rank", F.rank().over(w))
df.withColumn("row_num", F.row_number().over(w))
df.withColumn("prev_value", F.lag("value", 1).over(w))
df.withColumn("next_value", F.lead("value", 1).over(w))
w_range = Window.partitionBy("category").orderBy("date").rowsBetween(-6, 0)
df.withColumn("rolling_7d_avg", F.avg("value").over(w_range))
Joins
df1.join(df2, on="key", how="inner") # inner, left, right, outer/full, cross, left_semi, left_anti
df1.join(df2, df1.id1 == df2.id2, how="left")
df1.join(F.broadcast(df2), on="key") # broadcast join hint for small df2 — avoids a shuffle
df1.join(df2, on="key", how="left_semi") # rows in df1 that HAVE a match (filter-style join)
df1.join(df2, on="key", how="left_anti") # rows in df1 with NO match
SQL Interface
df.createOrReplaceTempView("my_table")
spark.sql("SELECT category, SUM(value) as total FROM my_table GROUP BY category").show()
# Mixing SQL and DataFrame API freely is common and idiomatic in PySpark
User-Defined Functions (UDFs)
from pyspark.sql.types import IntegerType
# Regular UDF — row-by-row Python execution, relatively slow (serialization overhead)
@F.udf(returnType=IntegerType())
def add_one(x):
return x + 1
df.withColumn("b", add_one("a"))
# Pandas UDF (vectorized) — MUCH faster, operates on batches via Arrow
import pandas as pd
from pyspark.sql.functions import pandas_udf
@pandas_udf(IntegerType())
def add_one_vectorized(s: pd.Series) -> pd.Series:
return s + 1
df.withColumn("b", add_one_vectorized("a"))
# Prefer built-in F.* functions over UDFs whenever possible —
# they run in the JVM without Python serialization overhead at all.
Repartitioning & Performance
df.repartition(200) # full shuffle to N partitions
df.repartition(10, "category") # hash-partition by column
df.coalesce(10) # reduce partitions WITHOUT a full shuffle (no increase)
df.cache() # persist in memory across actions
df.persist(StorageLevel.MEMORY_AND_DISK)
df.unpersist()
spark.conf.set("spark.sql.shuffle.partitions", "200") # default post-shuffle partition count
spark.conf.set("spark.sql.adaptive.enabled", "true") # adaptive query execution (AQE)
df.explain(mode="cost") # inspect cost-based plan
Common Column Functions
F.col("a")
F.lit(5) # literal value as a column
F.when(cond, val).otherwise(default)
F.coalesce(F.col("a"), F.col("b"), F.lit(0)) # first non-null
F.concat(F.col("a"), F.lit("_"), F.col("b"))
F.concat_ws("-", "a", "b", "c")
F.split(F.col("a"), ",")
F.explode(F.col("list_col")) # unnest array column into rows
F.array(F.col("a"), F.col("b"))
F.struct(F.col("a"), F.col("b"))
F.to_json(F.col("struct_col"))
F.from_json(F.col("json_col"), schema)
F.regexp_extract(F.col("a"), r"(\d+)", 1)
F.regexp_replace(F.col("a"), "old", "new")
F.date_add(F.col("date"), 7)
F.datediff(F.col("end"), F.col("start"))
F.date_format(F.col("date"), "yyyy-MM")
F.current_date(); F.current_timestamp()
RDD Basics (lower-level API, rarely needed directly)
rdd = spark.sparkContext.parallelize([1, 2, 3, 4])
rdd.map(lambda x: x * 2).collect()
rdd.filter(lambda x: x > 2).collect()
rdd.reduce(lambda a, b: a + b)
rdd.flatMap(lambda x: [x, x*2]).collect()
df.rdd.map(lambda row: row.asDict()) # DataFrame -> RDD when needed
Common Gotchas & Tips
- Spark is lazy — transformations (
select,filter,withColumn, etc.) build a plan; only actions (show,count,collect,write) trigger execution. .collect()pulls all data to the driver — never call it on large DataFrames; use.show(),.take(n), or write to storage instead.- Avoid Python UDFs when a built-in
F.*function exists — UDFs serialize data between JVM and Python, which is slow. Prefer Pandas UDFs over row-at-a-time UDFs when a UDF is unavoidable. - Watch for data skew in joins/group-bys — a few oversized partitions can dominate runtime; consider salting keys or broadcast joins for small tables.
df.count()triggers a full job — avoid calling it repeatedly just to check progress.- Use
explain()to check whether a join is a broadcast join, shuffle hash join, or sort-merge join when debugging performance. - Enable Adaptive Query Execution (
spark.sql.adaptive.enabled=true, default on in modern Spark) to let Spark re-optimize shuffle partitions and joins at runtime.