In [2]:
import findspark
findspark.init()
In [7]:
from pyspark.sql import SparkSession

spark = SparkSession \
    .builder \
    .appName("Miscellaneous Operations") \
    .getOrCreate()

Setting up the Data in Pyspark

In [8]:
valuesA = [('John', 100000), ('James', 150000), ('Emily', 65000), ('Nina', 200000)]

tableA = spark.createDataFrame(valuesA, ['name', 'salary'])
In [9]:
tableA.show()
+-----+------+
| name|salary|
+-----+------+
| John|100000|
|James|150000|
|Emily| 65000|
| Nina|200000|
+-----+------+

In [10]:
valuesB = [('James', 2), ('Emily',3), ('Darth Vader', 5), ('Princess Leia', 6),]

tableB = spark.createDataFrame(valuesB, ['name', 'employee_id'])
In [11]:
tableB.show()
+-------------+-----------+
|         name|employee_id|
+-------------+-----------+
|        James|          2|
|        Emily|          3|
|  Darth Vader|          5|
|Princess Leia|          6|
+-------------+-----------+

Inner join

In [12]:
inner_join = tableA.join(tableB, tableA.name == tableB.name)
inner_join.show()
+-----+------+-----+-----------+
| name|salary| name|employee_id|
+-----+------+-----+-----------+
|James|150000|James|          2|
|Emily| 65000|Emily|          3|
+-----+------+-----+-----------+

Left outer join

In [15]:
left_join = tableA.join(tableB, tableA.name == tableB.name, how='left') 
left_join.show()
+-----+------+-----+-----------+
| name|salary| name|employee_id|
+-----+------+-----+-----------+
|James|150000|James|          2|
| John|100000| null|       null|
|Emily| 65000|Emily|          3|
| Nina|200000| null|       null|
+-----+------+-----+-----------+

Right outer join

In [16]:
right_join = tableA.join(tableB, tableA.name == tableB.name, how='right') 
right_join.show()
+-----+------+-------------+-----------+
| name|salary|         name|employee_id|
+-----+------+-------------+-----------+
|James|150000|        James|          2|
| null|  null|Princess Leia|          6|
|Emily| 65000|        Emily|          3|
| null|  null|  Darth Vader|          5|
+-----+------+-------------+-----------+

Full outer join

In [17]:
full_outer_join = tableA.join(tableB, tableA.name == tableB.name, how='full')
full_outer_join.show()
+-----+------+-------------+-----------+
| name|salary|         name|employee_id|
+-----+------+-------------+-----------+
|James|150000|        James|          2|
| John|100000|         null|       null|
| null|  null|Princess Leia|          6|
|Emily| 65000|        Emily|          3|
| Nina|200000|         null|       null|
| null|  null|  Darth Vader|          5|
+-----+------+-------------+-----------+

In [ ]: