From 4811ee54e20bad985e0627ca940d3f6d2158e141 Mon Sep 17 00:00:00 2001 From: sabaalmas <67426913+sabaalmas@users.noreply.github.com> Date: Wed, 1 Dec 2021 21:47:04 -0500 Subject: [PATCH] Created using Colaboratory --- A_5_Spark.ipynb | 616 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 616 insertions(+) create mode 100644 A_5_Spark.ipynb diff --git a/A_5_Spark.ipynb b/A_5_Spark.ipynb new file mode 100644 index 00000000..d331d2ed --- /dev/null +++ b/A_5_Spark.ipynb @@ -0,0 +1,616 @@ +{ + "nbformat": 4, + "nbformat_minor": 0, + "metadata": { + "colab": { + "name": "A 5 Spark.ipynb", + "provenance": [], + "collapsed_sections": [], + "authorship_tag": "ABX9TyN29tyMptfu/LYJzcrV5WHQ", + "include_colab_link": true + }, + "kernelspec": { + "name": "python3", + "display_name": "Python 3" + }, + "language_info": { + "name": "python" + } + }, + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "id": "view-in-github", + "colab_type": "text" + }, + "source": [ + "\"Open" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "Cej1eaquba6E" + }, + "source": [ + "# Name : Almas Saba\n", + "# Student id: 40156359\n", + "\n", + "!apt-get install openjdk-8-jdk-headless -qq > /dev/null\n", + "\n", + "\n", + "!wget -q https://dlcdn.apache.org/spark/spark-3.2.0/spark-3.2.0-bin-hadoop3.2.tgz\n", + "\n", + "\n", + "!tar xf spark-3.2.0-bin-hadoop3.2.tgz\n", + "\n", + "\n", + "!pip install -q findspark\n", + "\n", + "import os\n", + "os.environ[\"JAVA_HOME\"] = \"/usr/lib/jvm/java-8-openjdk-amd64\"\n", + "os.environ[\"SPARK_HOME\"] = \"/content/spark-3.2.0-bin-hadoop3.2\"" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "GhPixa4YcVUB" + }, + "source": [ + "import findspark\n", + "findspark.init()" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 219 + }, + "id": "-fS5KIUEcVbK", + "outputId": "548bfcb4-2965-44ef-a8ac-e19df07ace6e" + }, + "source": [ + "from pyspark.sql import SparkSession\n", + "\n", + "spark = SparkSession.builder.master(\"local\").appName(\"APP\").getOrCreate()\n", + "spark" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/html": [ + "\n", + "
\n", + "

SparkSession - in-memory

\n", + " \n", + "
\n", + "

SparkContext

\n", + "\n", + "

Spark UI

\n", + "\n", + "
\n", + "
Version
\n", + "
v3.2.0
\n", + "
Master
\n", + "
local
\n", + "
AppName
\n", + "
APP
\n", + "
\n", + "
\n", + " \n", + "
\n", + " " + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "execution_count": 3 + } + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "-O3PYikwcVdy" + }, + "source": [ + "from pyspark.sql import functions as F" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "-_UVGokrcVhO" + }, + "source": [ + "M_df = spark.read.csv(\"/content/movies.csv\", header=True, inferSchema=True)" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "yTLj5F4ccqEF" + }, + "source": [ + "M_df2 = M_df.filter(F.col(\"genres\").contains(\"Drama\"))" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "NvizN9uncw4h", + "outputId": "8e087b69-d0c7-41fc-c89e-4c82036b77f7" + }, + "source": [ + "print(\"Total number of movies falling under genre Drama are :\"\n", + " ,M_df2.count())" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Total number of movies falling under genre Drama are : 4361\n" + ] + } + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "rCxEnP4_cw_w", + "outputId": "2ff805e0-c9c1-4e2c-8fbb-acd2e4f257cb" + }, + "source": [ + "from pyspark.sql import functions as F\n", + "\n", + "ratings_df = spark.read.csv(\"/content/ratings.csv\", header=True, inferSchema=True)\n", + "ratings_df.where(F.col(\"rating\").isNotNull()).show()\n", + "ratings_df.select(\"movieId\").distinct().show()\n", + "ratings_df.select(\"movieId\").distinct().count()" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "+------+-------+------+---------+\n", + "|userId|movieId|rating|timestamp|\n", + "+------+-------+------+---------+\n", + "| 1| 1| 4.0|964982703|\n", + "| 1| 3| 4.0|964981247|\n", + "| 1| 6| 4.0|964982224|\n", + "| 1| 47| 5.0|964983815|\n", + "| 1| 50| 5.0|964982931|\n", + "| 1| 70| 3.0|964982400|\n", + "| 1| 101| 5.0|964980868|\n", + "| 1| 110| 4.0|964982176|\n", + "| 1| 151| 5.0|964984041|\n", + "| 1| 157| 5.0|964984100|\n", + "| 1| 163| 5.0|964983650|\n", + "| 1| 216| 5.0|964981208|\n", + "| 1| 223| 3.0|964980985|\n", + "| 1| 231| 5.0|964981179|\n", + "| 1| 235| 4.0|964980908|\n", + "| 1| 260| 5.0|964981680|\n", + "| 1| 296| 3.0|964982967|\n", + "| 1| 316| 3.0|964982310|\n", + "| 1| 333| 5.0|964981179|\n", + "| 1| 349| 4.0|964982563|\n", + "+------+-------+------+---------+\n", + "only showing top 20 rows\n", + "\n", + "+-------+\n", + "|movieId|\n", + "+-------+\n", + "| 1580|\n", + "| 2366|\n", + "| 3175|\n", + "| 1088|\n", + "| 32460|\n", + "| 44022|\n", + "| 96488|\n", + "| 1238|\n", + "| 1342|\n", + "| 1591|\n", + "| 1645|\n", + "| 4519|\n", + "| 2142|\n", + "| 471|\n", + "| 3997|\n", + "| 833|\n", + "| 3918|\n", + "| 7982|\n", + "| 1959|\n", + "| 68135|\n", + "+-------+\n", + "only showing top 20 rows\n", + "\n" + ] + }, + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "9724" + ] + }, + "metadata": {}, + "execution_count": 17 + } + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "pVfUxHTWdAFx", + "outputId": "b95013b2-1049-487b-a6e3-805a09ca30c5" + }, + "source": [ + "##. 3. Who gave the most ratings, how many rates did he make?\n", + "from pyspark.sql import functions as F\n", + "\n", + "\n", + "rate_df = spark.read.csv(\"ratings.csv\", header=True, inferSchema=True)\n", + "rate_df.cache()\n", + "rate_df2 = rate_df.groupby('userId').count().sort('count', ascending=False)\n", + "rate_df2.first()" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "Row(userId=414, count=2698)" + ] + }, + "metadata": {}, + "execution_count": 11 + } + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "GMtS1ybldAI8", + "outputId": "7ab91d55-932f-496f-f966-0d0b42a218a0" + }, + "source": [ + "##. 4. Compute min, average, max rating per movie.\n", + "from pyspark.sql import functions as F\n", + "\n", + "df = spark.read.csv(\"ratings.csv\", header=True, inferSchema=True)\n", + "df.cache()\n", + "df.groupBy(\"movieId\").agg(F.avg('rating'), F.min('rating'),F.max('rating')).sort('movieId', ascending=True).show()" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "+-------+------------------+-----------+-----------+\n", + "|movieId| avg(rating)|min(rating)|max(rating)|\n", + "+-------+------------------+-----------+-----------+\n", + "| 1|3.9209302325581397| 0.5| 5.0|\n", + "| 2|3.4318181818181817| 0.5| 5.0|\n", + "| 3|3.2596153846153846| 0.5| 5.0|\n", + "| 4| 2.357142857142857| 1.0| 3.0|\n", + "| 5|3.0714285714285716| 0.5| 5.0|\n", + "| 6| 3.946078431372549| 1.0| 5.0|\n", + "| 7| 3.185185185185185| 1.0| 5.0|\n", + "| 8| 2.875| 1.0| 5.0|\n", + "| 9| 3.125| 1.5| 5.0|\n", + "| 10| 3.496212121212121| 0.5| 5.0|\n", + "| 11|3.6714285714285713| 1.0| 5.0|\n", + "| 12|2.4210526315789473| 1.0| 5.0|\n", + "| 13| 3.125| 2.0| 4.0|\n", + "| 14|3.8333333333333335| 3.0| 5.0|\n", + "| 15| 3.0| 1.0| 5.0|\n", + "| 16| 3.926829268292683| 1.0| 5.0|\n", + "| 17|3.7761194029850746| 0.5| 5.0|\n", + "| 18| 3.7| 2.0| 5.0|\n", + "| 19| 2.727272727272727| 1.0| 5.0|\n", + "| 20| 2.5| 1.0| 4.0|\n", + "+-------+------------------+-----------+-----------+\n", + "only showing top 20 rows\n", + "\n" + ] + } + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "OJTHK60X_Ivz", + "outputId": "fcc88dd5-b90f-4082-ecbb-a3c1d994ed28" + }, + "source": [ + "##. 5. Output dataset containing users that have rated a movie but not tagged it.\n", + "from pyspark.sql import functions as F\n", + "\n", + "df_ratings = spark.read.csv(\"ratings.csv\", header=True, inferSchema=True).select('userId','movieId','rating')\n", + "df_tags = spark.read.csv(\"tags.csv\", header=True, inferSchema=True).select('userId','movieId','tag')\n", + "df_ratings = df_ratings.join(df_tags, on=['userId','movieId'], how='left')\n", + "df_ratings.filter((df_ratings.rating.isNotNull()) & (df_ratings.tag.isNull())).show()" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "+------+-------+------+----+\n", + "|userId|movieId|rating| tag|\n", + "+------+-------+------+----+\n", + "| 1| 1| 4.0|null|\n", + "| 1| 3| 4.0|null|\n", + "| 1| 6| 4.0|null|\n", + "| 1| 47| 5.0|null|\n", + "| 1| 50| 5.0|null|\n", + "| 1| 70| 3.0|null|\n", + "| 1| 101| 5.0|null|\n", + "| 1| 110| 4.0|null|\n", + "| 1| 151| 5.0|null|\n", + "| 1| 157| 5.0|null|\n", + "| 1| 163| 5.0|null|\n", + "| 1| 216| 5.0|null|\n", + "| 1| 223| 3.0|null|\n", + "| 1| 231| 5.0|null|\n", + "| 1| 235| 4.0|null|\n", + "| 1| 260| 5.0|null|\n", + "| 1| 296| 3.0|null|\n", + "| 1| 316| 3.0|null|\n", + "| 1| 333| 5.0|null|\n", + "| 1| 349| 4.0|null|\n", + "+------+-------+------+----+\n", + "only showing top 20 rows\n", + "\n" + ] + } + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "wkLkG-yo_PN-", + "outputId": "5cf1e8e4-fac3-4940-e484-c8d06394f2b6" + }, + "source": [ + "##. 6. Output dataset containing users that have rated AND tagged a movie.\n", + "from pyspark.sql import functions as F\n", + "\n", + "df_ratings = spark.read.csv(\"ratings.csv\", header=True, inferSchema=True).select('userId','movieId','rating')\n", + "df_tags = spark.read.csv(\"tags.csv\", header=True, inferSchema=True).select('userId','movieId','tag')\n", + "df_ratings = df_ratings.join(df_tags, on=['userId','movieId'], how='left')\n", + "df_ratings.filter((df_ratings.rating.isNotNull()) & (df_ratings.tag.isNotNull())).show()" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "+------+-------+------+-----------------+\n", + "|userId|movieId|rating| tag|\n", + "+------+-------+------+-----------------+\n", + "| 2| 60756| 5.0| will ferrell|\n", + "| 2| 60756| 5.0| Highly quotable|\n", + "| 2| 60756| 5.0| funny|\n", + "| 2| 89774| 5.0| Tom Hardy|\n", + "| 2| 89774| 5.0| MMA|\n", + "| 2| 89774| 5.0| Boxing story|\n", + "| 2| 106782| 5.0| Martin Scorsese|\n", + "| 2| 106782| 5.0|Leonardo DiCaprio|\n", + "| 2| 106782| 5.0| drugs|\n", + "| 7| 48516| 1.0| way too long|\n", + "| 18| 431| 4.0| mafia|\n", + "| 18| 431| 4.0| gangster|\n", + "| 18| 431| 4.0| Al Pacino|\n", + "| 18| 1221| 5.0| Mafia|\n", + "| 18| 1221| 5.0| Al Pacino|\n", + "| 18| 5995| 4.5| true story|\n", + "| 18| 5995| 4.5| holocaust|\n", + "| 18| 44665| 4.5| twist ending|\n", + "| 18| 52604| 4.5| twist ending|\n", + "| 18| 52604| 4.5| courtroom drama|\n", + "+------+-------+------+-----------------+\n", + "only showing top 20 rows\n", + "\n" + ] + } + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "HU_3Lf5D_RZn", + "outputId": "03b0bc5d-985c-4736-ff49-0e66aba195a2" + }, + "source": [ + "##. 7. Output dataset showing the number of movies per Genre per Year (movies will be counted many times if it's associated with multiple genres).\n", + "from pyspark.sql import functions as F\n", + "\n", + "\n", + "df_movies = spark.read.csv(\"movies.csv\", header=True, inferSchema=True)\\\n", + " .withColumn('year', F.regexp_extract(F.col('title'), '\\(\\d{4}\\)', 0))\\\n", + " .withColumn('year', F.regexp_replace('year', '[\\(),]', ''))\\\n", + " .select('year','genres')\n", + "\n", + "df_movies = df_movies.withColumn('genres',F.explode(F.split(F.col(\"genres\"), \"\\|+\")))\\\n", + " .filter(df_movies.year != 0)\\\n", + " .groupBy('year','genres').count().sort('year','genres')\n", + "\n", + "df_movies.show(100)" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "+----+-----------+-----+\n", + "|year| genres|count|\n", + "+----+-----------+-----+\n", + "|1902| Action| 1|\n", + "|1902| Adventure| 1|\n", + "|1902| Fantasy| 1|\n", + "|1902| Sci-Fi| 1|\n", + "|1903| Crime| 1|\n", + "|1903| Western| 1|\n", + "|1908| Animation| 1|\n", + "|1908| Comedy| 1|\n", + "|1908| Sci-Fi| 1|\n", + "|1915| Drama| 1|\n", + "|1915| War| 1|\n", + "|1916| Action| 1|\n", + "|1916| Adventure| 1|\n", + "|1916| Comedy| 1|\n", + "|1916| Drama| 1|\n", + "|1916| Fantasy| 1|\n", + "|1916| Romance| 1|\n", + "|1916| Sci-Fi| 1|\n", + "|1917| Comedy| 1|\n", + "|1919| Comedy| 1|\n", + "|1919| Drama| 1|\n", + "|1920| Comedy| 1|\n", + "|1920| Crime| 1|\n", + "|1920| Fantasy| 1|\n", + "|1920| Horror| 1|\n", + "|1921| Comedy| 1|\n", + "|1921| Drama| 1|\n", + "|1922| Horror| 1|\n", + "|1923| Action| 1|\n", + "|1923| Comedy| 2|\n", + "|1923| Drama| 2|\n", + "|1923| Horror| 1|\n", + "|1923| Romance| 1|\n", + "|1924| Action| 2|\n", + "|1924| Adventure| 2|\n", + "|1924| Comedy| 2|\n", + "|1924| Drama| 2|\n", + "|1924| Fantasy| 3|\n", + "|1924| Romance| 2|\n", + "|1924| Sci-Fi| 1|\n", + "|1924| Thriller| 1|\n", + "|1925| Adventure| 2|\n", + "|1925| Comedy| 2|\n", + "|1925| Drama| 2|\n", + "|1925| Romance| 2|\n", + "|1925| War| 1|\n", + "|1926| Adventure| 1|\n", + "|1926| Comedy| 2|\n", + "|1926| Drama| 3|\n", + "|1926| Fantasy| 1|\n", + "|1926| Horror| 1|\n", + "|1926| Romance| 1|\n", + "|1926| War| 1|\n", + "|1927| Comedy| 1|\n", + "|1927| Crime| 1|\n", + "|1927| Drama| 5|\n", + "|1927| Horror| 1|\n", + "|1927| Musical| 1|\n", + "|1927| Mystery| 1|\n", + "|1927| Romance| 2|\n", + "|1927| Sci-Fi| 1|\n", + "|1927| Thriller| 1|\n", + "|1928| Animation| 1|\n", + "|1928| Children| 1|\n", + "|1928| Comedy| 3|\n", + "|1928| Drama| 1|\n", + "|1928| Musical| 1|\n", + "|1928| Romance| 1|\n", + "|1929| Comedy| 1|\n", + "|1929|Documentary| 1|\n", + "|1929| Fantasy| 1|\n", + "|1929| Musical| 2|\n", + "|1930| Action| 1|\n", + "|1930| Comedy| 1|\n", + "|1930| Drama| 4|\n", + "|1930| Musical| 1|\n", + "|1930| Romance| 1|\n", + "|1930| War| 1|\n", + "|1931| Action| 1|\n", + "|1931| Comedy| 4|\n", + "|1931| Crime| 3|\n", + "|1931| Drama| 9|\n", + "|1931| Film-Noir| 1|\n", + "|1931| Horror| 3|\n", + "|1931| Musical| 1|\n", + "|1931| Mystery| 1|\n", + "|1931| Romance| 3|\n", + "|1931| Sci-Fi| 1|\n", + "|1931| Thriller| 1|\n", + "|1931| Western| 1|\n", + "|1932| Comedy| 3|\n", + "|1932| Crime| 3|\n", + "|1932| Drama| 4|\n", + "|1932| Film-Noir| 1|\n", + "|1932| Horror| 2|\n", + "|1932| Romance| 4|\n", + "|1932| War| 1|\n", + "|1933| Action| 1|\n", + "|1933| Adventure| 2|\n", + "|1933| Children| 1|\n", + "+----+-----------+-----+\n", + "only showing top 100 rows\n", + "\n" + ] + } + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "rjfYI_ZW_Wzq" + }, + "source": [ + "" + ], + "execution_count": null, + "outputs": [] + } + ] +} \ No newline at end of file