Inferred and explicit schemas

In [1]:
import findspark
findspark.init()
In [2]:
import pyspark
from pyspark import SparkContext
sc = SparkContext.getOrCreate()
In [3]:
from pyspark.sql import SparkSession

spark = SparkSession \
    .builder \
    .appName("Inferred and explicit schemas") \
    .getOrCreate()
In [4]:
from pyspark.sql.types import Row

Inferring schema

In [5]:
lines = sc.textFile("../datasets/students.txt")
In [6]:
lines.collect()
Out[6]:
['Emily,44,55,78', 'Andy,47,34,89', 'Rick,55,78,55', 'Aaron,66,34,98']
In [10]:
parts = lines.map(lambda l: l.split(","))

parts.collect()
Out[10]:
[['Emily', '44', '55', '78'],
 ['Andy', '47', '34', '89'],
 ['Rick', '55', '78', '55'],
 ['Aaron', '66', '34', '98']]
In [11]:
students = parts.map(lambda p: Row(name=p[0], math=int(p[1]), english=int(p[2]), science=int(p[3])))
In [12]:
students.collect()
Out[12]:
[Row(english=55, math=44, name='Emily', science=78),
 Row(english=34, math=47, name='Andy', science=89),
 Row(english=78, math=55, name='Rick', science=55),
 Row(english=34, math=66, name='Aaron', science=98)]
In [13]:
schemaStudents = spark.createDataFrame(students)

schemaStudents.createOrReplaceTempView("students")
In [15]:
schemaStudents.columns
Out[15]:
['english', 'math', 'name', 'science']
In [16]:
schemaStudents.schema
Out[16]:
StructType(List(StructField(english,LongType,true),StructField(math,LongType,true),StructField(name,StringType,true),StructField(science,LongType,true)))
In [17]:
spark.sql("SELECT * FROM students").show()
+-------+----+-----+-------+
|english|math| name|science|
+-------+----+-----+-------+
|     55|  44|Emily|     78|
|     34|  47| Andy|     89|
|     78|  55| Rick|     55|
|     34|  66|Aaron|     98|
+-------+----+-----+-------+

Explicit schema

In [18]:
parts.collect()
Out[18]:
[['Emily', '44', '55', '78'],
 ['Andy', '47', '34', '89'],
 ['Rick', '55', '78', '55'],
 ['Aaron', '66', '34', '98']]
In [20]:
schemaString = "name math english science"
In [21]:
from pyspark.sql.types import StructType, StructField, StringType, LongType

fields = [StructField('name', StringType(), True),
          StructField('math', LongType(), True),
          StructField('english', LongType(), True),
          StructField('science', LongType(), True),
]
In [22]:
schema = StructType(fields)
In [23]:
schemaStudents = spark.createDataFrame(parts, schema)
In [24]:
schemaStudents.columns
Out[24]:
['name', 'math', 'english', 'science']
In [25]:
schemaStudents.schema
Out[25]:
StructType(List(StructField(name,StringType,true),StructField(math,LongType,true),StructField(english,LongType,true),StructField(science,LongType,true)))
In [26]:
spark.sql("SELECT * FROM students").show()
+-------+----+-----+-------+
|english|math| name|science|
+-------+----+-----+-------+
|     55|  44|Emily|     78|
|     34|  47| Andy|     89|
|     78|  55| Rick|     55|
|     34|  66|Aaron|     98|
+-------+----+-----+-------+

In [ ]: