我想计算总数的百分比,并将其保存在pyspark dataframe行的新列中:

spark不是这样工作的,你首先要收集数据,然后用它来计算百分比。下面是相同的示例代码,还有其他方法,一种是这样:

sum_spend = df.agg(F.sum(F.col("total_spend")).alias("sum_spend")).collect()[0][0]

df.withColumn(
    "needed",
    F.concat((F.col("total_spend")*100.0/F.lit(sum_spend)), F.lit("%"))
).show()

一种可能性:

import org.apache.spark.sql.expressions._
import org.apache.spark.sql.functions._

val df = sc.parallelize(Seq(
   ("A", "X", 2, 100), ("A", "X", 7, 100), ("B", "X", 10, 100),
   ("C", "X", 1, 100), ("D", "X", 50, 100), ("E", "X", 30, 100)
    )).toDF("c1", "c2", "Val1", "Val2")

val df2 = df
   .groupBy("c1")
   .agg(sum("Val1").alias("sum"))
   .withColumn("fraction", col("sum") /  sum("sum").over())

 df2.show

请在下面找到答案。

df.withColumn("needed",concat(df.col("total_spend").multiply(100)/df.agg(sum(col("total_spend"))).first.mkString.toInt,lit("%"))).show()