Lets create a dataframe from list of row object . First populate the list with row object and then we create the structfield and add it to the list. Pass the list into the createStructType function and pass this into the createDataFrame function.
import java.util.ArrayList; import java.util.Arrays; import java.util.List; import org.apache.spark.sql.Dataset; import org.apache.spark.sql.Row; import org.apache.spark.sql.RowFactory; import org.apache.spark.sql.SparkSession; import org.apache.spark.sql.types.DataTypes; import org.apache.spark.sql.types.StringType; public class DataFrameDocumentation { public static void main(String[] args) { SparkSession spark = SparkSession.builder(). appName("documentation").master("local").getOrCreate(); spark.sparkContext().setLogLevel("ERROR"); List<Row> list=new ArrayList<Row>(); list.add(RowFactory.create("one")); list.add(RowFactory.create("two")); list.add(RowFactory.create("three")); list.add(RowFactory.create("four")); List<org.apache.spark.sql.types.StructField> listOfStructField= new ArrayList<org.apache.spark.sql.types.StructField>(); listOfStructField.add (DataTypes.createStructField("test", DataTypes.StringType, true)); StructType structType=DataTypes.createStructType(listOfStructField); Dataset<Row> data=spark.createDataFrame(list,structType); data.show(); //Lets create the dataset of row using the Arrays asList Function Dataset<Row> test= spark.createDataFrame(Arrays.asList( new Movie("movie1",2323d,"1212"), new Movie("movie2",2323d,"1212"), new Movie("movie3",2323d,"1212"), new Movie("movie4",2323d,"1212") ), Movie.class); test.show();
Below is the Movie class used in the above code
import java.io.Serializable; public class Movie implements Serializable { private String name; private Double rating; private String timestamp; public Movie(String name, Double rating, String timestamp) { super(); this.name = name; this.rating = rating; this.timestamp = timestamp; } public Movie() { } public String getName() { return name; } public void setName(String name) { this.name = name; } public Double getRating() { return rating; } public void setRating(Double rating) { this.rating = rating; } public String getTimestamp() { return timestamp; } public void setTimestamp(String timestamp) { this.timestamp = timestamp; } }