look_at_acdemoc_dataset.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pyspark as ps\n",
    "spark = (ps.sql.SparkSession.builder \n",
    "        .master(\"local[4]\") \n",
    "        .appName(\"yelp_academic\") \n",
    "        .getOrCreate()\n",
    "        )\n",
    "sc = spark.sparkContext"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "yelp_a_biz = spark.read.json('yelp_dataset/yelp_academic_dataset_business.json')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "yelp_rev = spark.read.json('yelp_dataset/yelp_academic_dataset_review.json')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "sc = spark.sparkContext"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "yelp_a_biz.createOrReplaceTempView('yelp_business')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "root\n",
      " |-- address: string (nullable = true)\n",
      " |-- attributes: struct (nullable = true)\n",
      " |    |-- AcceptsInsurance: string (nullable = true)\n",
      " |    |-- AgesAllowed: string (nullable = true)\n",
      " |    |-- Alcohol: string (nullable = true)\n",
      " |    |-- Ambience: string (nullable = true)\n",
      " |    |-- BYOB: string (nullable = true)\n",
      " |    |-- BYOBCorkage: string (nullable = true)\n",
      " |    |-- BestNights: string (nullable = true)\n",
      " |    |-- BikeParking: string (nullable = true)\n",
      " |    |-- BusinessAcceptsBitcoin: string (nullable = true)\n",
      " |    |-- BusinessAcceptsCreditCards: string (nullable = true)\n",
      " |    |-- BusinessParking: string (nullable = true)\n",
      " |    |-- ByAppointmentOnly: string (nullable = true)\n",
      " |    |-- Caters: string (nullable = true)\n",
      " |    |-- CoatCheck: string (nullable = true)\n",
      " |    |-- Corkage: string (nullable = true)\n",
      " |    |-- DietaryRestrictions: string (nullable = true)\n",
      " |    |-- DogsAllowed: string (nullable = true)\n",
      " |    |-- DriveThru: string (nullable = true)\n",
      " |    |-- GoodForDancing: string (nullable = true)\n",
      " |    |-- GoodForKids: string (nullable = true)\n",
      " |    |-- GoodForMeal: string (nullable = true)\n",
      " |    |-- HairSpecializesIn: string (nullable = true)\n",
      " |    |-- HappyHour: string (nullable = true)\n",
      " |    |-- HasTV: string (nullable = true)\n",
      " |    |-- Music: string (nullable = true)\n",
      " |    |-- NoiseLevel: string (nullable = true)\n",
      " |    |-- Open24Hours: string (nullable = true)\n",
      " |    |-- OutdoorSeating: string (nullable = true)\n",
      " |    |-- RestaurantsAttire: string (nullable = true)\n",
      " |    |-- RestaurantsCounterService: string (nullable = true)\n",
      " |    |-- RestaurantsDelivery: string (nullable = true)\n",
      " |    |-- RestaurantsGoodForGroups: string (nullable = true)\n",
      " |    |-- RestaurantsPriceRange2: string (nullable = true)\n",
      " |    |-- RestaurantsReservations: string (nullable = true)\n",
      " |    |-- RestaurantsTableService: string (nullable = true)\n",
      " |    |-- RestaurantsTakeOut: string (nullable = true)\n",
      " |    |-- Smoking: string (nullable = true)\n",
      " |    |-- WheelchairAccessible: string (nullable = true)\n",
      " |    |-- WiFi: string (nullable = true)\n",
      " |-- business_id: string (nullable = true)\n",
      " |-- categories: string (nullable = true)\n",
      " |-- city: string (nullable = true)\n",
      " |-- hours: struct (nullable = true)\n",
      " |    |-- Friday: string (nullable = true)\n",
      " |    |-- Monday: string (nullable = true)\n",
      " |    |-- Saturday: string (nullable = true)\n",
      " |    |-- Sunday: string (nullable = true)\n",
      " |    |-- Thursday: string (nullable = true)\n",
      " |    |-- Tuesday: string (nullable = true)\n",
      " |    |-- Wednesday: string (nullable = true)\n",
      " |-- is_open: long (nullable = true)\n",
      " |-- latitude: double (nullable = true)\n",
      " |-- longitude: double (nullable = true)\n",
      " |-- name: string (nullable = true)\n",
      " |-- neighborhood: string (nullable = true)\n",
      " |-- postal_code: string (nullable = true)\n",
      " |-- review_count: long (nullable = true)\n",
      " |-- stars: double (nullable = true)\n",
      " |-- state: string (nullable = true)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "yelp_a_biz.printSchema()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------------------+--------------------+-----+--------------------+\n",
      "|         business_id|                name|stars|          categories|\n",
      "+--------------------+--------------------+-----+--------------------+\n",
      "|CeuTRtwsq6w5rztGO...|            Salsitas|  2.5|Mexican, Restaurants|\n",
      "|Zkesf4VRamyOBswJA...|       Audio Express|  3.5|Automotive, Auto ...|\n",
      "|Ju2tEB-zllh9FoCma...|Above and Beyond ...|  3.5|Pets, Pet Groomer...|\n",
      "|27fSAl_GsDDhrxzpE...|    Jiffy Lube #1664|  3.0|Oil Change Statio...|\n",
      "|yBf7mMFELRa1vRora...|        Desert Homes|  2.0|Apartments, Home ...|\n",
      "|WnRtn80rdsWOa2nWZ...|Central Phoenix W...|  3.5|Obstetricians & G...|\n",
      "|8vA1d9_w4hBjOcrM7...|       Boston Market|  3.5|American (Traditi...|\n",
      "|xnJl1DcMULTTzY21I...|Verizon Authorize...|  3.5|Professional Serv...|\n",
      "|RQOorijTwEMC7G7Mi...|Integrity Air Con...|  4.0|Home Services, He...|\n",
      "|GWpKZSrf3kO25vby0...|Valencia Park Apa...|  2.5|Real Estate, Apar...|\n",
      "+--------------------+--------------------+-----+--------------------+\n",
      "only showing top 10 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "select = '''\n",
    "SELECT business_id, name, stars, categories\n",
    "FROM yelp_business \n",
    "WHERE  city = 'Phoenix' and review_count > 10 and review_count < 40 \n",
    "\n",
    "\n",
    "'''\n",
    "phoe_gr_40 = spark.sql(select)\n",
    "phoe_gr_40.show(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "phoe_biz = phoe_gr_40.toPandas()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(5475, 4)"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "root\n",
      " |-- business_id: string (nullable = true)\n",
      " |-- cool: long (nullable = true)\n",
      " |-- date: string (nullable = true)\n",
      " |-- funny: long (nullable = true)\n",
      " |-- review_id: string (nullable = true)\n",
      " |-- stars: long (nullable = true)\n",
      " |-- text: string (nullable = true)\n",
      " |-- useful: long (nullable = true)\n",
      " |-- user_id: string (nullable = true)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "yelp_rev.printSchema()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "bid = phoe_biz.business_id[:3]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['45bWSZtniwPRiqlivpS8Og', '_c3ixq9jYKxhLUB0czi0ug', 'cKRMmytHxaSt8F0SMEzKqg']"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "list(bid)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "rviews = yelp_rev.select('business_id', 'user_id', 'stars')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "pho_rv = rviews.filter(rviews.business_id.isin(list(phoe_biz.business_id)))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "111814"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pho_rv.count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[Row(business_id='Szcr-yMxq76zRaSTUapAcw', user_id='0pf5VuzE4_1pwj5NJHG5TQ', stars=3),\n",
       " Row(business_id='BNzcfz3jmBo_1wUB5YIsXg', user_id='0pf5VuzE4_1pwj5NJHG5TQ', stars=4)]"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pho_rv.take(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "-131021924558"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hash('0pf5VuzE4_1pwj5NJHG5TQ')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "hash_udf = udf(lambda idx: hash(idx) , IntegerType())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "pho_rv.withColumn?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Row(business_id='Szcr-yMxq76zRaSTUapAcw', user_id='0pf5VuzE4_1pwj5NJHG5TQ', stars=3, user_num=-1178701720, biz_num=-759057679)"
      ]
     },
     "execution_count": 67,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pho_rv_n = pho_rv.withColumn('user_num', hash_udf(pho_rv.user_id))\n",
    "pho_rv_nn = pho_rv_n.withColumn('biz_num', hash_udf(pho_rv.business_id))\n",
    "\n",
    "pho_rv_nn.first()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [],
   "source": [
    "splits = pho_rv_nn.randomSplit([0.7,0.3], seed =91)\n",
    "train = splits[0]\n",
    "test = splits[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(['business_id', 'user_id', 'stars', 'user_num', 'biz_num'],\n",
       " ['business_id', 'user_id', 'stars', 'user_num', 'biz_num'])"
      ]
     },
     "execution_count": 71,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.columns, test.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pyspark.ml.recommendation import ALS\n",
    "from pyspark.sql.types import IntegerType\n",
    "from pyspark.sql.functions import udf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [],
   "source": [
    "als = ALS(rank=10, regParam=0.1, \n",
    "          itemCol='biz_num', \n",
    "          userCol='user_num',\n",
    "          ratingCol='stars')\n",
    "mdl = als.fit(train)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_res = mdl.transform(train)\n",
    "test_res = mdl.transform(test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pyspark.ml.evaluation import RegressionEvaluator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Row(business_id='KCVnCb6TvTzoDp4t-Qf4wQ', user_id='8JAx8FnnLFZVc0h5W71zHw', stars=5, user_num=-1070722737, biz_num=-2024711065, prediction=4.839774131774902)"
      ]
     },
     "execution_count": 75,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_res.first()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_res.na.drop?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [],
   "source": [
    "evaluator = RegressionEvaluator(labelCol=\"stars\",predictionCol=\"prediction\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "### Have to create some basic metric"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_res1 = test_res.na.drop()\n",
    "test_res2 = test_res.fillna(3.6)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(3.57183013844712, 0.12047564570355435)"
      ]
     },
     "execution_count": 100,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    " evaluator.evaluate(train_res)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(3.5718301384471194, 2.705844144986854)"
      ]
     },
     "execution_count": 103,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "evaluator.evaluate(test_res1), evaluator.evaluate(test_res2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_res3 = test_res.withColumn?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_res3 = test_res.withColumn('prediction', test_res.prediction*0+ 3.6)\n",
    "test_res3 = test_res3.fillna(3.6)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1.6644441298302348"
      ]
     },
     "execution_count": 113,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "evaluator.evaluate(test_res3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[Row(avg(stars)=3.6376998518669867)]"
      ]
     },
     "execution_count": 83,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.agg({'stars':'mean'}).collect()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "To create model first I have to filter all bussiness and all reviews by category  'restaurants' and may be selected cities.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 117,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "yelp_a_biz.filter(yelp_a_biz.city == 'San Francisco' ).count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {},
   "outputs": [],
   "source": [
    "yelp_a_biz.filter?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "ename": "AnalysisException",
     "evalue": "'java.lang.RuntimeException: java.lang.RuntimeException: Unable to instantiate org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClient;'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mPy4JJavaError\u001b[0m                             Traceback (most recent call last)",
      "\u001b[0;32m/usr/local/Cellar/apache-spark/2.3.2/libexec/python/pyspark/sql/utils.py\u001b[0m in \u001b[0;36mdeco\u001b[0;34m(*a, **kw)\u001b[0m\n\u001b[1;32m     62\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 63\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0ma\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkw\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     64\u001b[0m         \u001b[0;32mexcept\u001b[0m \u001b[0mpy4j\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mprotocol\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mPy4JJavaError\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/Cellar/apache-spark/2.3.2/libexec/python/lib/py4j-0.10.7-src.zip/py4j/protocol.py\u001b[0m in \u001b[0;36mget_return_value\u001b[0;34m(answer, gateway_client, target_id, name)\u001b[0m\n\u001b[1;32m    327\u001b[0m                     \u001b[0;34m\"An error occurred while calling {0}{1}{2}.\\n\"\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 328\u001b[0;31m                     format(target_id, \".\", name), value)\n\u001b[0m\u001b[1;32m    329\u001b[0m             \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mPy4JJavaError\u001b[0m: An error occurred while calling o34.sql.\n: org.apache.spark.sql.AnalysisException: java.lang.RuntimeException: java.lang.RuntimeException: Unable to instantiate org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClient;\n\tat org.apache.spark.sql.hive.HiveExternalCatalog.withClient(HiveExternalCatalog.scala:106)\n\tat org.apache.spark.sql.hive.HiveExternalCatalog.databaseExists(HiveExternalCatalog.scala:194)\n\tat org.apache.spark.sql.internal.SharedState.externalCatalog$lzycompute(SharedState.scala:114)\n\tat org.apache.spark.sql.internal.SharedState.externalCatalog(SharedState.scala:102)\n\tat org.apache.spark.sql.hive.HiveSessionStateBuilder.externalCatalog(HiveSessionStateBuilder.scala:39)\n\tat org.apache.spark.sql.hive.HiveSessionStateBuilder.catalog$lzycompute(HiveSessionStateBuilder.scala:54)\n\tat org.apache.spark.sql.hive.HiveSessionStateBuilder.catalog(HiveSessionStateBuilder.scala:52)\n\tat org.apache.spark.sql.hive.HiveSessionStateBuilder$$anon$1.<init>(HiveSessionStateBuilder.scala:69)\n\tat org.apache.spark.sql.hive.HiveSessionStateBuilder.analyzer(HiveSessionStateBuilder.scala:69)\n\tat org.apache.spark.sql.internal.BaseSessionStateBuilder$$anonfun$build$2.apply(BaseSessionStateBuilder.scala:293)\n\tat org.apache.spark.sql.internal.BaseSessionStateBuilder$$anonfun$build$2.apply(BaseSessionStateBuilder.scala:293)\n\tat org.apache.spark.sql.internal.SessionState.analyzer$lzycompute(SessionState.scala:79)\n\tat org.apache.spark.sql.internal.SessionState.analyzer(SessionState.scala:79)\n\tat org.apache.spark.sql.execution.QueryExecution.analyzed$lzycompute(QueryExecution.scala:57)\n\tat org.apache.spark.sql.execution.QueryExecution.analyzed(QueryExecution.scala:55)\n\tat org.apache.spark.sql.execution.QueryExecution.assertAnalyzed(QueryExecution.scala:47)\n\tat org.apache.spark.sql.Dataset$.ofRows(Dataset.scala:74)\n\tat org.apache.spark.sql.SparkSession.sql(SparkSession.scala:642)\n\tat sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\n\tat sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\n\tat sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\n\tat java.lang.reflect.Method.invoke(Method.java:498)\n\tat py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\n\tat py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357)\n\tat py4j.Gateway.invoke(Gateway.java:282)\n\tat py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\n\tat py4j.commands.CallCommand.execute(CallCommand.java:79)\n\tat py4j.GatewayConnection.run(GatewayConnection.java:238)\n\tat java.lang.Thread.run(Thread.java:748)\nCaused by: java.lang.RuntimeException: java.lang.RuntimeException: Unable to instantiate org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClient\n\tat org.apache.hadoop.hive.ql.session.SessionState.start(SessionState.java:522)\n\tat org.apache.spark.sql.hive.client.HiveClientImpl.newState(HiveClientImpl.scala:180)\n\tat org.apache.spark.sql.hive.client.HiveClientImpl.<init>(HiveClientImpl.scala:114)\n\tat sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)\n\tat sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)\n\tat sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)\n\tat java.lang.reflect.Constructor.newInstance(Constructor.java:423)\n\tat org.apache.spark.sql.hive.client.IsolatedClientLoader.createClient(IsolatedClientLoader.scala:264)\n\tat org.apache.spark.sql.hive.HiveUtils$.newClientForMetadata(HiveUtils.scala:385)\n\tat org.apache.spark.sql.hive.HiveUtils$.newClientForMetadata(HiveUtils.scala:287)\n\tat org.apache.spark.sql.hive.HiveExternalCatalog.client$lzycompute(HiveExternalCatalog.scala:66)\n\tat org.apache.spark.sql.hive.HiveExternalCatalog.client(HiveExternalCatalog.scala:65)\n\tat org.apache.spark.sql.hive.HiveExternalCatalog$$anonfun$databaseExists$1.apply$mcZ$sp(HiveExternalCatalog.scala:195)\n\tat org.apache.spark.sql.hive.HiveExternalCatalog$$anonfun$databaseExists$1.apply(HiveExternalCatalog.scala:195)\n\tat org.apache.spark.sql.hive.HiveExternalCatalog$$anonfun$databaseExists$1.apply(HiveExternalCatalog.scala:195)\n\tat org.apache.spark.sql.hive.HiveExternalCatalog.withClient(HiveExternalCatalog.scala:97)\n\t... 28 more\nCaused by: java.lang.RuntimeException: Unable to instantiate org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClient\n\tat org.apache.hadoop.hive.metastore.MetaStoreUtils.newInstance(MetaStoreUtils.java:1523)\n\tat org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.<init>(RetryingMetaStoreClient.java:86)\n\tat org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:132)\n\tat org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:104)\n\tat org.apache.hadoop.hive.ql.metadata.Hive.createMetaStoreClient(Hive.java:3005)\n\tat org.apache.hadoop.hive.ql.metadata.Hive.getMSC(Hive.java:3024)\n\tat org.apache.hadoop.hive.ql.session.SessionState.start(SessionState.java:503)\n\t... 43 more\nCaused by: java.lang.reflect.InvocationTargetException\n\tat sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)\n\tat sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)\n\tat sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)\n\tat java.lang.reflect.Constructor.newInstance(Constructor.java:423)\n\tat org.apache.hadoop.hive.metastore.MetaStoreUtils.newInstance(MetaStoreUtils.java:1521)\n\t... 49 more\nCaused by: javax.jdo.JDOFatalDataStoreException: Unable to open a test connection to the given database. JDBC url = jdbc:derby:;databaseName=metastore_db;create=true, username = APP. Terminating connection pool (set lazyInit to true if you expect to start your database after your app). Original Exception: ------\r\njava.sql.SQLException: Failed to start database 'metastore_db' with class loader org.apache.spark.sql.hive.client.IsolatedClientLoader$$anon$1@506f25ba, see the next exception for details.\n\tat org.apache.derby.impl.jdbc.SQLExceptionFactory.getSQLException(Unknown Source)\n\tat org.apache.derby.impl.jdbc.SQLExceptionFactory.getSQLException(Unknown Source)\n\tat org.apache.derby.impl.jdbc.Util.seeNextException(Unknown Source)\n\tat org.apache.derby.impl.jdbc.EmbedConnection.bootDatabase(Unknown Source)\n\tat org.apache.derby.impl.jdbc.EmbedConnection.<init>(Unknown Source)\n\tat org.apache.derby.jdbc.InternalDriver$1.run(Unknown Source)\n\tat org.apache.derby.jdbc.InternalDriver$1.run(Unknown Source)\n\tat java.security.AccessController.doPrivileged(Native Method)\n\tat org.apache.derby.jdbc.InternalDriver.getNewEmbedConnection(Unknown Source)\n\tat org.apache.derby.jdbc.InternalDriver.connect(Unknown Source)\n\tat org.apache.derby.jdbc.InternalDriver.connect(Unknown Source)\n\tat org.apache.derby.jdbc.AutoloadedDriver.connect(Unknown Source)\n\tat java.sql.DriverManager.getConnection(DriverManager.java:664)\n\tat java.sql.DriverManager.getConnection(DriverManager.java:208)\n\tat com.jolbox.bonecp.BoneCP.obtainRawInternalConnection(BoneCP.java:361)\n\tat com.jolbox.bonecp.BoneCP.<init>(BoneCP.java:416)\n\tat com.jolbox.bonecp.BoneCPDataSource.getConnection(BoneCPDataSource.java:120)\n\tat org.datanucleus.store.rdbms.ConnectionFactoryImpl$ManagedConnectionImpl.getConnection(ConnectionFactoryImpl.java:501)\n\tat org.datanucleus.store.rdbms.RDBMSStoreManager.<init>(RDBMSStoreManager.java:298)\n\tat sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)\n\tat sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)\n\tat sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)\n\tat java.lang.reflect.Constructor.newInstance(Constructor.java:423)\n\tat org.datanucleus.plugin.NonManagedPluginRegistry.createExecutableExtension(NonManagedPluginRegistry.java:631)\n\tat org.datanucleus.plugin.PluginManager.createExecutableExtension(PluginManager.java:301)\n\tat org.datanucleus.NucleusContext.createStoreManagerForProperties(NucleusContext.java:1187)\n\tat org.datanucleus.NucleusContext.initialise(NucleusContext.java:356)\n\tat org.datanucleus.api.jdo.JDOPersistenceManagerFactory.freezeConfiguration(JDOPersistenceManagerFactory.java:775)\n\tat org.datanucleus.api.jdo.JDOPersistenceManagerFactory.createPersistenceManagerFactory(JDOPersistenceManagerFactory.java:333)\n\tat org.datanucleus.api.jdo.JDOPersistenceManagerFactory.getPersistenceManagerFactory(JDOPersistenceManagerFactory.java:202)\n\tat sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\n\tat sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\n\tat sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\n\tat java.lang.reflect.Method.invoke(Method.java:498)\n\tat javax.jdo.JDOHelper$16.run(JDOHelper.java:1965)\n\tat java.security.AccessController.doPrivileged(Native Method)\n\tat javax.jdo.JDOHelper.invoke(JDOHelper.java:1960)\n\tat javax.jdo.JDOHelper.invokeGetPersistenceManagerFactoryOnImplementation(JDOHelper.java:1166)\n\tat javax.jdo.JDOHelper.getPersistenceManagerFactory(JDOHelper.java:808)\n\tat javax.jdo.JDOHelper.getPersistenceManagerFactory(JDOHelper.java:701)\n\tat org.apache.hadoop.hive.metastore.ObjectStore.getPMF(ObjectStore.java:365)\n\tat org.apache.hadoop.hive.metastore.ObjectStore.getPersistenceManager(ObjectStore.java:394)\n\tat org.apache.hadoop.hive.metastore.ObjectStore.initialize(ObjectStore.java:291)\n\tat org.apache.hadoop.hive.metastore.ObjectStore.setConf(ObjectStore.java:258)\n\tat org.apache.hadoop.util.ReflectionUtils.setConf(ReflectionUtils.java:76)\n\tat org.apache.hadoop.util.ReflectionUtils.newInstance(ReflectionUtils.java:136)\n\tat org.apache.hadoop.hive.metastore.RawStoreProxy.<init>(RawStoreProxy.java:57)\n\tat org.apache.hadoop.hive.metastore.RawStoreProxy.getProxy(RawStoreProxy.java:66)\n\tat org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.newRawStore(HiveMetaStore.java:593)\n\tat org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.getMS(HiveMetaStore.java:571)\n\tat org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.createDefaultDB(HiveMetaStore.java:624)\n\tat org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.init(HiveMetaStore.java:461)\n\tat org.apache.hadoop.hive.metastore.RetryingHMSHandler.<init>(RetryingHMSHandler.java:66)\n\tat org.apache.hadoop.hive.metastore.RetryingHMSHandler.getProxy(RetryingHMSHandler.java:72)\n\tat org.apache.hadoop.hive.metastore.HiveMetaStore.newRetryingHMSHandler(HiveMetaStore.java:5762)\n\tat org.apache.hadoop.hive.metastore.HiveMetaStoreClient.<init>(HiveMetaStoreClient.java:199)\n\tat org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClient.<init>(SessionHiveMetaStoreClient.java:74)\n\tat sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)\n\tat sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)\n\tat sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)\n\tat java.lang.reflect.Constructor.newInstance(Constructor.java:423)\n\tat org.apache.hadoop.hive.metastore.MetaStoreUtils.newInstance(MetaStoreUtils.java:1521)\n\tat org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.<init>(RetryingMetaStoreClient.java:86)\n\tat org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:132)\n\tat org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:104)\n\tat org.apache.hadoop.hive.ql.metadata.Hive.createMetaStoreClient(Hive.java:3005)\n\tat org.apache.hadoop.hive.ql.metadata.Hive.getMSC(Hive.java:3024)\n\tat org.apache.hadoop.hive.ql.session.SessionState.start(SessionState.java:503)\n\tat org.apache.spark.sql.hive.client.HiveClientImpl.newState(HiveClientImpl.scala:180)\n\tat org.apache.spark.sql.hive.client.HiveClientImpl.<init>(HiveClientImpl.scala:114)\n\tat sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)\n\tat sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)\n\tat sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)\n\tat java.lang.reflect.Constructor.newInstance(Constructor.java:423)\n\tat org.apache.spark.sql.hive.client.IsolatedClientLoader.createClient(IsolatedClientLoader.scala:264)\n\tat org.apache.spark.sql.hive.HiveUtils$.newClientForMetadata(HiveUtils.scala:385)\n\tat org.apache.spark.sql.hive.HiveUtils$.newClientForMetadata(HiveUtils.scala:287)\n\tat org.apache.spark.sql.hive.HiveExternalCatalog.client$lzycompute(HiveExternalCatalog.scala:66)\n\tat org.apache.spark.sql.hive.HiveExternalCatalog.client(HiveExternalCatalog.scala:65)\n\tat org.apache.spark.sql.hive.HiveExternalCatalog$$anonfun$databaseExists$1.apply$mcZ$sp(HiveExternalCatalog.scala:195)\n\tat org.apache.spark.sql.hive.HiveExternalCatalog$$anonfun$databaseExists$1.apply(HiveExternalCatalog.scala:195)\n\tat org.apache.spark.sql.hive.HiveExternalCatalog$$anonfun$databaseExists$1.apply(HiveExternalCatalog.scala:195)\n\tat org.apache.spark.sql.hive.HiveExternalCatalog.withClient(HiveExternalCatalog.scala:97)\n\tat org.apache.spark.sql.hive.HiveExternalCatalog.databaseExists(HiveExternalCatalog.scala:194)\n\tat org.apache.spark.sql.internal.SharedState.externalCatalog$lzycompute(SharedState.scala:114)\n\tat org.apache.spark.sql.internal.SharedState.externalCatalog(SharedState.scala:102)\n\tat org.apache.spark.sql.hive.HiveSessionStateBuilder.externalCatalog(HiveSessionStateBuilder.scala:39)\n\tat org.apache.spark.sql.hive.HiveSessionStateBuilder.catalog$lzycompute(HiveSessionStateBuilder.scala:54)\n\tat org.apache.spark.sql.hive.HiveSessionStateBuilder.catalog(HiveSessionStateBuilder.scala:52)\n\tat org.apache.spark.sql.hive.HiveSessionStateBuilder$$anon$1.<init>(HiveSessionStateBuilder.scala:69)\n\tat org.apache.spark.sql.hive.HiveSessionStateBuilder.analyzer(HiveSessionStateBuilder.scala:69)\n\tat org.apache.spark.sql.internal.BaseSessionStateBuilder$$anonfun$build$2.apply(BaseSessionStateBuilder.scala:293)\n\tat org.apache.spark.sql.internal.BaseSessionStateBuilder$$anonfun$build$2.apply(BaseSessionStateBuilder.scala:293)\n\tat org.apache.spark.sql.internal.SessionState.analyzer$lzycompute(SessionState.scala:79)\n\tat org.apache.spark.sql.internal.SessionState.analyzer(SessionState.scala:79)\n\tat org.apache.spark.sql.execution.QueryExecution.analyzed$lzycompute(QueryExecution.scala:57)\n\tat org.apache.spark.sql.execution.QueryExecution.analyzed(QueryExecution.scala:55)\n\tat org.apache.spark.sql.execution.QueryExecution.assertAnalyzed(QueryExecution.scala:47)\n\tat org.apache.spark.sql.Dataset$.ofRows(Dataset.scala:74)\n\tat org.apache.spark.sql.SparkSession.sql(SparkSession.scala:642)\n\tat sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\n\tat sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\n\tat sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\n\tat java.lang.reflect.Method.invoke(Method.java:498)\n\tat py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\n\tat py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357)\n\tat py4j.Gateway.invoke(Gateway.java:282)\n\tat py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\n\tat py4j.commands.CallCommand.execute(CallCommand.java:79)\n\tat py4j.GatewayConnection.run(GatewayConnection.java:238)\n\tat java.lang.Thread.run(Thread.java:748)\nCaused by: ERROR XJ040: Failed to start database 'metastore_db' with class loader org.apache.spark.sql.hive.client.IsolatedClientLoader$$anon$1@506f25ba, see the next exception for details.\n\tat org.apache.derby.iapi.error.StandardException.newException(Unknown Source)\n\tat org.apache.derby.impl.jdbc.SQLExceptionFactory.wrapArgsForTransportAcrossDRDA(Unknown Source)\n\t... 111 more\nCaused by: ERROR XSDB6: Another instance of Derby may have already booted the database /Users/olga/yelper/metastore_db.\n\tat org.apache.derby.iapi.error.StandardException.newException(Unknown Source)\n\tat org.apache.derby.iapi.error.StandardException.newException(Unknown Source)\n\tat org.apache.derby.impl.store.raw.data.BaseDataFileFactory.privGetJBMSLockOnDB(Unknown Source)\n\tat org.apache.derby.impl.store.raw.data.BaseDataFileFactory.run(Unknown Source)\n\tat java.security.AccessController.doPrivileged(Native Method)\n\tat org.apache.derby.impl.store.raw.data.BaseDataFileFactory.getJBMSLockOnDB(Unknown Source)\n\tat org.apache.derby.impl.store.raw.data.BaseDataFileFactory.boot(Unknown Source)\n\tat org.apache.derby.impl.services.monitor.BaseMonitor.boot(Unknown Source)\n\tat org.apache.derby.impl.services.monitor.TopService.bootModule(Unknown Source)\n\tat org.apache.derby.impl.services.monitor.BaseMonitor.startModule(Unknown Source)\n\tat org.apache.derby.impl.services.monitor.FileMonitor.startModule(Unknown Source)\n\tat org.apache.derby.iapi.services.monitor.Monitor.bootServiceModule(Unknown Source)\n\tat org.apache.derby.impl.store.raw.RawStore$6.run(Unknown Source)\n\tat java.security.AccessController.doPrivileged(Native Method)\n\tat org.apache.derby.impl.store.raw.RawStore.bootServiceModule(Unknown Source)\n\tat org.apache.derby.impl.store.raw.RawStore.boot(Unknown Source)\n\tat org.apache.derby.impl.services.monitor.BaseMonitor.boot(Unknown Source)\n\tat org.apache.derby.impl.services.monitor.TopService.bootModule(Unknown Source)\n\tat org.apache.derby.impl.services.monitor.BaseMonitor.startModule(Unknown Source)\n\tat org.apache.derby.impl.services.monitor.FileMonitor.startModule(Unknown Source)\n\tat org.apache.derby.iapi.services.monitor.Monitor.bootServiceModule(Unknown Source)\n\tat org.apache.derby.impl.store.access.RAMAccessManager$5.run(Unknown Source)\n\tat java.security.AccessController.doPrivileged(Native Method)\n\tat org.apache.derby.impl.store.access.RAMAccessManager.bootServiceModule(Unknown Source)\n\tat org.apache.derby.impl.store.access.RAMAccessManager.boot(Unknown Source)\n\tat org.apache.derby.impl.services.monitor.BaseMonitor.boot(Unknown Source)\n\tat org.apache.derby.impl.services.monitor.TopService.bootModule(Unknown Source)\n\tat org.apache.derby.impl.services.monitor.BaseMonitor.startModule(Unknown Source)\n\tat org.apache.derby.impl.services.monitor.FileMonitor.startModule(Unknown Source)\n\tat org.apache.derby.iapi.services.monitor.Monitor.bootServiceModule(Unknown Source)\n\tat org.apache.derby.impl.db.BasicDatabase$5.run(Unknown Source)\n\tat java.security.AccessController.doPrivileged(Native Method)\n\tat org.apache.derby.impl.db.BasicDatabase.bootServiceModule(Unknown Source)\n\tat org.apache.derby.impl.db.BasicDatabase.bootStore(Unknown Source)\n\tat org.apache.derby.impl.db.BasicDatabase.boot(Unknown Source)\n\tat org.apache.derby.impl.services.monitor.BaseMonitor.boot(Unknown Source)\n\tat org.apache.derby.impl.services.monitor.TopService.bootModule(Unknown Source)\n\tat org.apache.derby.impl.services.monitor.BaseMonitor.bootService(Unknown Source)\n\tat org.apache.derby.impl.services.monitor.BaseMonitor.startProviderService(Unknown Source)\n\tat org.apache.derby.impl.services.monitor.BaseMonitor.findProviderAndStartService(Unknown Source)\n\tat org.apache.derby.impl.services.monitor.BaseMonitor.startPersistentService(Unknown Source)\n\tat org.apache.derby.iapi.services.monitor.Monitor.startPersistentService(Unknown Source)\n\tat org.apache.derby.impl.jdbc.EmbedConnection$4.run(Unknown Source)\n\tat org.apache.derby.impl.jdbc.EmbedConnection$4.run(Unknown Source)\n\tat java.security.AccessController.doPrivileged(Native Method)\n\tat org.apache.derby.impl.jdbc.EmbedConnection.startPersistentService(Unknown Source)\n\t... 108 more\n------\r\n\nNestedThrowables:\njava.sql.SQLException: Unable to open a test connection to the given database. JDBC url = jdbc:derby:;databaseName=metastore_db;create=true, username = APP. Terminating connection pool (set lazyInit to true if you expect to start your database after your app). Original Exception: ------\r\njava.sql.SQLException: Failed to start database 'metastore_db' with class loader org.apache.spark.sql.hive.client.IsolatedClientLoader$$anon$1@506f25ba, see the next exception for details.\n\tat org.apache.derby.impl.jdbc.SQLExceptionFactory.getSQLException(Unknown Source)\n\tat org.apache.derby.impl.jdbc.SQLExceptionFactory.getSQLException(Unknown Source)\n\tat org.apache.derby.impl.jdbc.Util.seeNextException(Unknown Source)\n\tat org.apache.derby.impl.jdbc.EmbedConnection.bootDatabase(Unknown Source)\n\tat org.apache.derby.impl.jdbc.EmbedConnection.<init>(Unknown Source)\n\tat org.apache.derby.jdbc.InternalDriver$1.run(Unknown Source)\n\tat org.apache.derby.jdbc.InternalDriver$1.run(Unknown Source)\n\tat java.security.AccessController.doPrivileged(Native Method)\n\tat org.apache.derby.jdbc.InternalDriver.getNewEmbedConnection(Unknown Source)\n\tat org.apache.derby.jdbc.InternalDriver.connect(Unknown Source)\n\tat org.apache.derby.jdbc.InternalDriver.connect(Unknown Source)\n\tat org.apache.derby.jdbc.AutoloadedDriver.connect(Unknown Source)\n\tat java.sql.DriverManager.getConnection(DriverManager.java:664)\n\tat java.sql.DriverManager.getConnection(DriverManager.java:208)\n\tat com.jolbox.bonecp.BoneCP.obtainRawInternalConnection(BoneCP.java:361)\n\tat com.jolbox.bonecp.BoneCP.<init>(BoneCP.java:416)\n\tat com.jolbox.bonecp.BoneCPDataSource.getConnection(BoneCPDataSource.java:120)\n\tat org.datanucleus.store.rdbms.ConnectionFactoryImpl$ManagedConnectionImpl.getConnection(ConnectionFactoryImpl.java:501)\n\tat org.datanucleus.store.rdbms.RDBMSStoreManager.<init>(RDBMSStoreManager.java:298)\n\tat sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)\n\tat sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)\n\tat sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)\n\tat java.lang.reflect.Constructor.newInstance(Constructor.java:423)\n\tat org.datanucleus.plugin.NonManagedPluginRegistry.createExecutableExtension(NonManagedPluginRegistry.java:631)\n\tat org.datanucleus.plugin.PluginManager.createExecutableExtension(PluginManager.java:301)\n\tat org.datanucleus.NucleusContext.createStoreManagerForProperties(NucleusContext.java:1187)\n\tat org.datanucleus.NucleusContext.initialise(NucleusContext.java:356)\n\tat org.datanucleus.api.jdo.JDOPersistenceManagerFactory.freezeConfiguration(JDOPersistenceManagerFactory.java:775)\n\tat org.datanucleus.api.jdo.JDOPersistenceManagerFactory.createPersistenceManagerFactory(JDOPersistenceManagerFactory.java:333)\n\tat org.datanucleus.api.jdo.JDOPersistenceManagerFactory.getPersistenceManagerFactory(JDOPersistenceManagerFactory.java:202)\n\tat sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\n\tat sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\n\tat sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\n\tat java.lang.reflect.Method.invoke(Method.java:498)\n\tat javax.jdo.JDOHelper$16.run(JDOHelper.java:1965)\n\tat java.security.AccessController.doPrivileged(Native Method)\n\tat javax.jdo.JDOHelper.invoke(JDOHelper.java:1960)\n\tat javax.jdo.JDOHelper.invokeGetPersistenceManagerFactoryOnImplementation(JDOHelper.java:1166)\n\tat javax.jdo.JDOHelper.getPersistenceManagerFactory(JDOHelper.java:808)\n\tat javax.jdo.JDOHelper.getPersistenceManagerFactory(JDOHelper.java:701)\n\tat org.apache.hadoop.hive.metastore.ObjectStore.getPMF(ObjectStore.java:365)\n\tat org.apache.hadoop.hive.metastore.ObjectStore.getPersistenceManager(ObjectStore.java:394)\n\tat org.apache.hadoop.hive.metastore.ObjectStore.initialize(ObjectStore.java:291)\n\tat org.apache.hadoop.hive.metastore.ObjectStore.setConf(ObjectStore.java:258)\n\tat org.apache.hadoop.util.ReflectionUtils.setConf(ReflectionUtils.java:76)\n\tat org.apache.hadoop.util.ReflectionUtils.newInstance(ReflectionUtils.java:136)\n\tat org.apache.hadoop.hive.metastore.RawStoreProxy.<init>(RawStoreProxy.java:57)\n\tat org.apache.hadoop.hive.metastore.RawStoreProxy.getProxy(RawStoreProxy.java:66)\n\tat org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.newRawStore(HiveMetaStore.java:593)\n\tat org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.getMS(HiveMetaStore.java:571)\n\tat org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.createDefaultDB(HiveMetaStore.java:624)\n\tat org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.init(HiveMetaStore.java:461)\n\tat org.apache.hadoop.hive.metastore.RetryingHMSHandler.<init>(RetryingHMSHandler.java:66)\n\tat org.apache.hadoop.hive.metastore.RetryingHMSHandler.getProxy(RetryingHMSHandler.java:72)\n\tat org.apache.hadoop.hive.metastore.HiveMetaStore.newRetryingHMSHandler(HiveMetaStore.java:5762)\n\tat org.apache.hadoop.hive.metastore.HiveMetaStoreClient.<init>(HiveMetaStoreClient.java:199)\n\tat org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClient.<init>(SessionHiveMetaStoreClient.java:74)\n\tat sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)\n\tat sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)\n\tat sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)\n\tat java.lang.reflect.Constructor.newInstance(Constructor.java:423)\n\tat org.apache.hadoop.hive.metastore.MetaStoreUtils.newInstance(MetaStoreUtils.java:1521)\n\tat org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.<init>(RetryingMetaStoreClient.java:86)\n\tat org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:132)\n\tat org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:104)\n\tat org.apache.hadoop.hive.ql.metadata.Hive.createMetaStoreClient(Hive.java:3005)\n\tat org.apache.hadoop.hive.ql.metadata.Hive.getMSC(Hive.java:3024)\n\tat org.apache.hadoop.hive.ql.session.SessionState.start(SessionState.java:503)\n\tat org.apache.spark.sql.hive.client.HiveClientImpl.newState(HiveClientImpl.scala:180)\n\tat org.apache.spark.sql.hive.client.HiveClientImpl.<init>(HiveClientImpl.scala:114)\n\tat sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)\n\tat sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)\n\tat sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)\n\tat java.lang.reflect.Constructor.newInstance(Constructor.java:423)\n\tat org.apache.spark.sql.hive.client.IsolatedClientLoader.createClient(IsolatedClientLoader.scala:264)\n\tat org.apache.spark.sql.hive.HiveUtils$.newClientForMetadata(HiveUtils.scala:385)\n\tat org.apache.spark.sql.hive.HiveUtils$.newClientForMetadata(HiveUtils.scala:287)\n\tat org.apache.spark.sql.hive.HiveExternalCatalog.client$lzycompute(HiveExternalCatalog.scala:66)\n\tat org.apache.spark.sql.hive.HiveExternalCatalog.client(HiveExternalCatalog.scala:65)\n\tat org.apache.spark.sql.hive.HiveExternalCatalog$$anonfun$databaseExists$1.apply$mcZ$sp(HiveExternalCatalog.scala:195)\n\tat org.apache.spark.sql.hive.HiveExternalCatalog$$anonfun$databaseExists$1.apply(HiveExternalCatalog.scala:195)\n\tat org.apache.spark.sql.hive.HiveExternalCatalog$$anonfun$databaseExists$1.apply(HiveExternalCatalog.scala:195)\n\tat org.apache.spark.sql.hive.HiveExternalCatalog.withClient(HiveExternalCatalog.scala:97)\n\tat org.apache.spark.sql.hive.HiveExternalCatalog.databaseExists(HiveExternalCatalog.scala:194)\n\tat org.apache.spark.sql.internal.SharedState.externalCatalog$lzycompute(SharedState.scala:114)\n\tat org.apache.spark.sql.internal.SharedState.externalCatalog(SharedState.scala:102)\n\tat org.apache.spark.sql.hive.HiveSessionStateBuilder.externalCatalog(HiveSessionStateBuilder.scala:39)\n\tat org.apache.spark.sql.hive.HiveSessionStateBuilder.catalog$lzycompute(HiveSessionStateBuilder.scala:54)\n\tat org.apache.spark.sql.hive.HiveSessionStateBuilder.catalog(HiveSessionStateBuilder.scala:52)\n\tat org.apache.spark.sql.hive.HiveSessionStateBuilder$$anon$1.<init>(HiveSessionStateBuilder.scala:69)\n\tat org.apache.spark.sql.hive.HiveSessionStateBuilder.analyzer(HiveSessionStateBuilder.scala:69)\n\tat org.apache.spark.sql.internal.BaseSessionStateBuilder$$anonfun$build$2.apply(BaseSessionStateBuilder.scala:293)\n\tat org.apache.spark.sql.internal.BaseSessionStateBuilder$$anonfun$build$2.apply(BaseSessionStateBuilder.scala:293)\n\tat org.apache.spark.sql.internal.SessionState.analyzer$lzycompute(SessionState.scala:79)\n\tat org.apache.spark.sql.internal.SessionState.analyzer(SessionState.scala:79)\n\tat org.apache.spark.sql.execution.QueryExecution.analyzed$lzycompute(QueryExecution.scala:57)\n\tat org.apache.spark.sql.execution.QueryExecution.analyzed(QueryExecution.scala:55)\n\tat org.apache.spark.sql.execution.QueryExecution.assertAnalyzed(QueryExecution.scala:47)\n\tat org.apache.spark.sql.Dataset$.ofRows(Dataset.scala:74)\n\tat org.apache.spark.sql.SparkSession.sql(SparkSession.scala:642)\n\tat sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\n\tat sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\n\tat sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\n\tat java.lang.reflect.Method.invoke(Method.java:498)\n\tat py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\n\tat py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357)\n\tat py4j.Gateway.invoke(Gateway.java:282)\n\tat py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\n\tat py4j.commands.CallCommand.execute(CallCommand.java:79)\n\tat py4j.GatewayConnection.run(GatewayConnection.java:238)\n\tat java.lang.Thread.run(Thread.java:748)\nCaused by: ERROR XJ040: Failed to start database 'metastore_db' with class loader org.apache.spark.sql.hive.client.IsolatedClientLoader$$anon$1@506f25ba, see the next exception for details.\n\tat org.apache.derby.iapi.error.StandardException.newException(Unknown Source)\n\tat org.apache.derby.impl.jdbc.SQLExceptionFactory.wrapArgsForTransportAcrossDRDA(Unknown Source)\n\t... 111 more\nCaused by: ERROR XSDB6: Another instance of Derby may have already booted the database /Users/olga/yelper/metastore_db.\n\tat org.apache.derby.iapi.error.StandardException.newException(Unknown Source)\n\tat org.apache.derby.iapi.error.StandardException.newException(Unknown Source)\n\tat org.apache.derby.impl.store.raw.data.BaseDataFileFactory.privGetJBMSLockOnDB(Unknown Source)\n\tat org.apache.derby.impl.store.raw.data.BaseDataFileFactory.run(Unknown Source)\n\tat java.security.AccessController.doPrivileged(Native Method)\n\tat org.apache.derby.impl.store.raw.data.BaseDataFileFactory.getJBMSLockOnDB(Unknown Source)\n\tat org.apache.derby.impl.store.raw.data.BaseDataFileFactory.boot(Unknown Source)\n\tat org.apache.derby.impl.services.monitor.BaseMonitor.boot(Unknown Source)\n\tat org.apache.derby.impl.services.monitor.TopService.bootModule(Unknown Source)\n\tat org.apache.derby.impl.services.monitor.BaseMonitor.startModule(Unknown Source)\n\tat org.apache.derby.impl.services.monitor.FileMonitor.startModule(Unknown Source)\n\tat org.apache.derby.iapi.services.monitor.Monitor.bootServiceModule(Unknown Source)\n\tat org.apache.derby.impl.store.raw.RawStore$6.run(Unknown Source)\n\tat java.security.AccessController.doPrivileged(Native Method)\n\tat org.apache.derby.impl.store.raw.RawStore.bootServiceModule(Unknown Source)\n\tat org.apache.derby.impl.store.raw.RawStore.boot(Unknown Source)\n\tat org.apache.derby.impl.services.monitor.BaseMonitor.boot(Unknown Source)\n\tat org.apache.derby.impl.services.monitor.TopService.bootModule(Unknown Source)\n\tat org.apache.derby.impl.services.monitor.BaseMonitor.startModule(Unknown Source)\n\tat org.apache.derby.impl.services.monitor.FileMonitor.startModule(Unknown Source)\n\tat org.apache.derby.iapi.services.monitor.Monitor.bootServiceModule(Unknown Source)\n\tat org.apache.derby.impl.store.access.RAMAccessManager$5.run(Unknown Source)\n\tat java.security.AccessController.doPrivileged(Native Method)\n\tat org.apache.derby.impl.store.access.RAMAccessManager.bootServiceModule(Unknown Source)\n\tat org.apache.derby.impl.store.access.RAMAccessManager.boot(Unknown Source)\n\tat org.apache.derby.impl.services.monitor.BaseMonitor.boot(Unknown Source)\n\tat org.apache.derby.impl.services.monitor.TopService.bootModule(Unknown Source)\n\tat org.apache.derby.impl.services.monitor.BaseMonitor.startModule(Unknown Source)\n\tat org.apache.derby.impl.services.monitor.FileMonitor.startModule(Unknown Source)\n\tat org.apache.derby.iapi.services.monitor.Monitor.bootServiceModule(Unknown Source)\n\tat org.apache.derby.impl.db.BasicDatabase$5.run(Unknown Source)\n\tat java.security.AccessController.doPrivileged(Native Method)\n\tat org.apache.derby.impl.db.BasicDatabase.bootServiceModule(Unknown Source)\n\tat org.apache.derby.impl.db.BasicDatabase.bootStore(Unknown Source)\n\tat org.apache.derby.impl.db.BasicDatabase.boot(Unknown Source)\n\tat org.apache.derby.impl.services.monitor.BaseMonitor.boot(Unknown Source)\n\tat org.apache.derby.impl.services.monitor.TopService.bootModule(Unknown Source)\n\tat org.apache.derby.impl.services.monitor.BaseMonitor.bootService(Unknown Source)\n\tat org.apache.derby.impl.services.monitor.BaseMonitor.startProviderService(Unknown Source)\n\tat org.apache.derby.impl.services.monitor.BaseMonitor.findProviderAndStartService(Unknown Source)\n\tat org.apache.derby.impl.services.monitor.BaseMonitor.startPersistentService(Unknown Source)\n\tat org.apache.derby.iapi.services.monitor.Monitor.startPersistentService(Unknown Source)\n\tat org.apache.derby.impl.jdbc.EmbedConnection$4.run(Unknown Source)\n\tat org.apache.derby.impl.jdbc.EmbedConnection$4.run(Unknown Source)\n\tat java.security.AccessController.doPrivileged(Native Method)\n\tat org.apache.derby.impl.jdbc.EmbedConnection.startPersistentService(Unknown Source)\n\t... 108 more\n------\r\n\n\tat org.datanucleus.api.jdo.NucleusJDOHelper.getJDOExceptionForNucleusException(NucleusJDOHelper.java:436)\n\tat org.datanucleus.api.jdo.JDOPersistenceManagerFactory.freezeConfiguration(JDOPersistenceManagerFactory.java:788)\n\tat org.datanucleus.api.jdo.JDOPersistenceManagerFactory.createPersistenceManagerFactory(JDOPersistenceManagerFactory.java:333)\n\tat org.datanucleus.api.jdo.JDOPersistenceManagerFactory.getPersistenceManagerFactory(JDOPersistenceManagerFactory.java:202)\n\tat sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\n\tat sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\n\tat sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\n\tat java.lang.reflect.Method.invoke(Method.java:498)\n\tat javax.jdo.JDOHelper$16.run(JDOHelper.java:1965)\n\tat java.security.AccessController.doPrivileged(Native Method)\n\tat javax.jdo.JDOHelper.invoke(JDOHelper.java:1960)\n\tat javax.jdo.JDOHelper.invokeGetPersistenceManagerFactoryOnImplementation(JDOHelper.java:1166)\n\tat javax.jdo.JDOHelper.getPersistenceManagerFactory(JDOHelper.java:808)\n\tat javax.jdo.JDOHelper.getPersistenceManagerFactory(JDOHelper.java:701)\n\tat org.apache.hadoop.hive.metastore.ObjectStore.getPMF(ObjectStore.java:365)\n\tat org.apache.hadoop.hive.metastore.ObjectStore.getPersistenceManager(ObjectStore.java:394)\n\tat org.apache.hadoop.hive.metastore.ObjectStore.initialize(ObjectStore.java:291)\n\tat org.apache.hadoop.hive.metastore.ObjectStore.setConf(ObjectStore.java:258)\n\tat org.apache.hadoop.util.ReflectionUtils.setConf(ReflectionUtils.java:76)\n\tat org.apache.hadoop.util.ReflectionUtils.newInstance(ReflectionUtils.java:136)\n\tat org.apache.hadoop.hive.metastore.RawStoreProxy.<init>(RawStoreProxy.java:57)\n\tat org.apache.hadoop.hive.metastore.RawStoreProxy.getProxy(RawStoreProxy.java:66)\n\tat org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.newRawStore(HiveMetaStore.java:593)\n\tat org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.getMS(HiveMetaStore.java:571)\n\tat org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.createDefaultDB(HiveMetaStore.java:624)\n\tat org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.init(HiveMetaStore.java:461)\n\tat org.apache.hadoop.hive.metastore.RetryingHMSHandler.<init>(RetryingHMSHandler.java:66)\n\tat org.apache.hadoop.hive.metastore.RetryingHMSHandler.getProxy(RetryingHMSHandler.java:72)\n\tat org.apache.hadoop.hive.metastore.HiveMetaStore.newRetryingHMSHandler(HiveMetaStore.java:5762)\n\tat org.apache.hadoop.hive.metastore.HiveMetaStoreClient.<init>(HiveMetaStoreClient.java:199)\n\tat org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClient.<init>(SessionHiveMetaStoreClient.java:74)\n\t... 54 more\nCaused by: java.sql.SQLException: Unable to open a test connection to the given database. JDBC url = jdbc:derby:;databaseName=metastore_db;create=true, username = APP. Terminating connection pool (set lazyInit to true if you expect to start your database after your app). Original Exception: ------\r\njava.sql.SQLException: Failed to start database 'metastore_db' with class loader org.apache.spark.sql.hive.client.IsolatedClientLoader$$anon$1@506f25ba, see the next exception for details.\n\tat org.apache.derby.impl.jdbc.SQLExceptionFactory.getSQLException(Unknown Source)\n\tat org.apache.derby.impl.jdbc.SQLExceptionFactory.getSQLException(Unknown Source)\n\tat org.apache.derby.impl.jdbc.Util.seeNextException(Unknown Source)\n\tat org.apache.derby.impl.jdbc.EmbedConnection.bootDatabase(Unknown Source)\n\tat org.apache.derby.impl.jdbc.EmbedConnection.<init>(Unknown Source)\n\tat org.apache.derby.jdbc.InternalDriver$1.run(Unknown Source)\n\tat org.apache.derby.jdbc.InternalDriver$1.run(Unknown Source)\n\tat java.security.AccessController.doPrivileged(Native Method)\n\tat org.apache.derby.jdbc.InternalDriver.getNewEmbedConnection(Unknown Source)\n\tat org.apache.derby.jdbc.InternalDriver.connect(Unknown Source)\n\tat org.apache.derby.jdbc.InternalDriver.connect(Unknown Source)\n\tat org.apache.derby.jdbc.AutoloadedDriver.connect(Unknown Source)\n\tat java.sql.DriverManager.getConnection(DriverManager.java:664)\n\tat java.sql.DriverManager.getConnection(DriverManager.java:208)\n\tat com.jolbox.bonecp.BoneCP.obtainRawInternalConnection(BoneCP.java:361)\n\tat com.jolbox.bonecp.BoneCP.<init>(BoneCP.java:416)\n\tat com.jolbox.bonecp.BoneCPDataSource.getConnection(BoneCPDataSource.java:120)\n\tat org.datanucleus.store.rdbms.ConnectionFactoryImpl$ManagedConnectionImpl.getConnection(ConnectionFactoryImpl.java:501)\n\tat org.datanucleus.store.rdbms.RDBMSStoreManager.<init>(RDBMSStoreManager.java:298)\n\tat sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)\n\tat sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)\n\tat sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)\n\tat java.lang.reflect.Constructor.newInstance(Constructor.java:423)\n\tat org.datanucleus.plugin.NonManagedPluginRegistry.createExecutableExtension(NonManagedPluginRegistry.java:631)\n\tat org.datanucleus.plugin.PluginManager.createExecutableExtension(PluginManager.java:301)\n\tat org.datanucleus.NucleusContext.createStoreManagerForProperties(NucleusContext.java:1187)\n\tat org.datanucleus.NucleusContext.initialise(NucleusContext.java:356)\n\tat org.datanucleus.api.jdo.JDOPersistenceManagerFactory.freezeConfiguration(JDOPersistenceManagerFactory.java:775)\n\tat org.datanucleus.api.jdo.JDOPersistenceManagerFactory.createPersistenceManagerFactory(JDOPersistenceManagerFactory.java:333)\n\tat org.datanucleus.api.jdo.JDOPersistenceManagerFactory.getPersistenceManagerFactory(JDOPersistenceManagerFactory.java:202)\n\tat sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\n\tat sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\n\tat sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\n\tat java.lang.reflect.Method.invoke(Method.java:498)\n\tat javax.jdo.JDOHelper$16.run(JDOHelper.java:1965)\n\tat java.security.AccessController.doPrivileged(Native Method)\n\tat javax.jdo.JDOHelper.invoke(JDOHelper.java:1960)\n\tat javax.jdo.JDOHelper.invokeGetPersistenceManagerFactoryOnImplementation(JDOHelper.java:1166)\n\tat javax.jdo.JDOHelper.getPersistenceManagerFactory(JDOHelper.java:808)\n\tat javax.jdo.JDOHelper.getPersistenceManagerFactory(JDOHelper.java:701)\n\tat org.apache.hadoop.hive.metastore.ObjectStore.getPMF(ObjectStore.java:365)\n\tat org.apache.hadoop.hive.metastore.ObjectStore.getPersistenceManager(ObjectStore.java:394)\n\tat org.apache.hadoop.hive.metastore.ObjectStore.initialize(ObjectStore.java:291)\n\tat org.apache.hadoop.hive.metastore.ObjectStore.setConf(ObjectStore.java:258)\n\tat org.apache.hadoop.util.ReflectionUtils.setConf(ReflectionUtils.java:76)\n\tat org.apache.hadoop.util.ReflectionUtils.newInstance(ReflectionUtils.java:136)\n\tat org.apache.hadoop.hive.metastore.RawStoreProxy.<init>(RawStoreProxy.java:57)\n\tat org.apache.hadoop.hive.metastore.RawStoreProxy.getProxy(RawStoreProxy.java:66)\n\tat org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.newRawStore(HiveMetaStore.java:593)\n\tat org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.getMS(HiveMetaStore.java:571)\n\tat org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.createDefaultDB(HiveMetaStore.java:624)\n\tat org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.init(HiveMetaStore.java:461)\n\tat org.apache.hadoop.hive.metastore.RetryingHMSHandler.<init>(RetryingHMSHandler.java:66)\n\tat org.apache.hadoop.hive.metastore.RetryingHMSHandler.getProxy(RetryingHMSHandler.java:72)\n\tat org.apache.hadoop.hive.metastore.HiveMetaStore.newRetryingHMSHandler(HiveMetaStore.java:5762)\n\tat org.apache.hadoop.hive.metastore.HiveMetaStoreClient.<init>(HiveMetaStoreClient.java:199)\n\tat org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClient.<init>(SessionHiveMetaStoreClient.java:74)\n\tat sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)\n\tat sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)\n\tat sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)\n\tat java.lang.reflect.Constructor.newInstance(Constructor.java:423)\n\tat org.apache.hadoop.hive.metastore.MetaStoreUtils.newInstance(MetaStoreUtils.java:1521)\n\tat org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.<init>(RetryingMetaStoreClient.java:86)\n\tat org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:132)\n\tat org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:104)\n\tat org.apache.hadoop.hive.ql.metadata.Hive.createMetaStoreClient(Hive.java:3005)\n\tat org.apache.hadoop.hive.ql.metadata.Hive.getMSC(Hive.java:3024)\n\tat org.apache.hadoop.hive.ql.session.SessionState.start(SessionState.java:503)\n\tat org.apache.spark.sql.hive.client.HiveClientImpl.newState(HiveClientImpl.scala:180)\n\tat org.apache.spark.sql.hive.client.HiveClientImpl.<init>(HiveClientImpl.scala:114)\n\tat sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)\n\tat sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)\n\tat sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)\n\tat java.lang.reflect.Constructor.newInstance(Constructor.java:423)\n\tat org.apache.spark.sql.hive.client.IsolatedClientLoader.createClient(IsolatedClientLoader.scala:264)\n\tat org.apache.spark.sql.hive.HiveUtils$.newClientForMetadata(HiveUtils.scala:385)\n\tat org.apache.spark.sql.hive.HiveUtils$.newClientForMetadata(HiveUtils.scala:287)\n\tat org.apache.spark.sql.hive.HiveExternalCatalog.client$lzycompute(HiveExternalCatalog.scala:66)\n\tat org.apache.spark.sql.hive.HiveExternalCatalog.client(HiveExternalCatalog.scala:65)\n\tat org.apache.spark.sql.hive.HiveExternalCatalog$$anonfun$databaseExists$1.apply$mcZ$sp(HiveExternalCatalog.scala:195)\n\tat org.apache.spark.sql.hive.HiveExternalCatalog$$anonfun$databaseExists$1.apply(HiveExternalCatalog.scala:195)\n\tat org.apache.spark.sql.hive.HiveExternalCatalog$$anonfun$databaseExists$1.apply(HiveExternalCatalog.scala:195)\n\tat org.apache.spark.sql.hive.HiveExternalCatalog.withClient(HiveExternalCatalog.scala:97)\n\tat org.apache.spark.sql.hive.HiveExternalCatalog.databaseExists(HiveExternalCatalog.scala:194)\n\tat org.apache.spark.sql.internal.SharedState.externalCatalog$lzycompute(SharedState.scala:114)\n\tat org.apache.spark.sql.internal.SharedState.externalCatalog(SharedState.scala:102)\n\tat org.apache.spark.sql.hive.HiveSessionStateBuilder.externalCatalog(HiveSessionStateBuilder.scala:39)\n\tat org.apache.spark.sql.hive.HiveSessionStateBuilder.catalog$lzycompute(HiveSessionStateBuilder.scala:54)\n\tat org.apache.spark.sql.hive.HiveSessionStateBuilder.catalog(HiveSessionStateBuilder.scala:52)\n\tat org.apache.spark.sql.hive.HiveSessionStateBuilder$$anon$1.<init>(HiveSessionStateBuilder.scala:69)\n\tat org.apache.spark.sql.hive.HiveSessionStateBuilder.analyzer(HiveSessionStateBuilder.scala:69)\n\tat org.apache.spark.sql.internal.BaseSessionStateBuilder$$anonfun$build$2.apply(BaseSessionStateBuilder.scala:293)\n\tat org.apache.spark.sql.internal.BaseSessionStateBuilder$$anonfun$build$2.apply(BaseSessionStateBuilder.scala:293)\n\tat org.apache.spark.sql.internal.SessionState.analyzer$lzycompute(SessionState.scala:79)\n\tat org.apache.spark.sql.internal.SessionState.analyzer(SessionState.scala:79)\n\tat org.apache.spark.sql.execution.QueryExecution.analyzed$lzycompute(QueryExecution.scala:57)\n\tat org.apache.spark.sql.execution.QueryExecution.analyzed(QueryExecution.scala:55)\n\tat org.apache.spark.sql.execution.QueryExecution.assertAnalyzed(QueryExecution.scala:47)\n\tat org.apache.spark.sql.Dataset$.ofRows(Dataset.scala:74)\n\tat org.apache.spark.sql.SparkSession.sql(SparkSession.scala:642)\n\tat sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\n\tat sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\n\tat sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\n\tat java.lang.reflect.Method.invoke(Method.java:498)\n\tat py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\n\tat py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357)\n\tat py4j.Gateway.invoke(Gateway.java:282)\n\tat py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\n\tat py4j.commands.CallCommand.execute(CallCommand.java:79)\n\tat py4j.GatewayConnection.run(GatewayConnection.java:238)\n\tat java.lang.Thread.run(Thread.java:748)\nCaused by: ERROR XJ040: Failed to start database 'metastore_db' with class loader org.apache.spark.sql.hive.client.IsolatedClientLoader$$anon$1@506f25ba, see the next exception for details.\n\tat org.apache.derby.iapi.error.StandardException.newException(Unknown Source)\n\tat org.apache.derby.impl.jdbc.SQLExceptionFactory.wrapArgsForTransportAcrossDRDA(Unknown Source)\n\t... 111 more\nCaused by: ERROR XSDB6: Another instance of Derby may have already booted the database /Users/olga/yelper/metastore_db.\n\tat org.apache.derby.iapi.error.StandardException.newException(Unknown Source)\n\tat org.apache.derby.iapi.error.StandardException.newException(Unknown Source)\n\tat org.apache.derby.impl.store.raw.data.BaseDataFileFactory.privGetJBMSLockOnDB(Unknown Source)\n\tat org.apache.derby.impl.store.raw.data.BaseDataFileFactory.run(Unknown Source)\n\tat java.security.AccessController.doPrivileged(Native Method)\n\tat org.apache.derby.impl.store.raw.data.BaseDataFileFactory.getJBMSLockOnDB(Unknown Source)\n\tat org.apache.derby.impl.store.raw.data.BaseDataFileFactory.boot(Unknown Source)\n\tat org.apache.derby.impl.services.monitor.BaseMonitor.boot(Unknown Source)\n\tat org.apache.derby.impl.services.monitor.TopService.bootModule(Unknown Source)\n\tat org.apache.derby.impl.services.monitor.BaseMonitor.startModule(Unknown Source)\n\tat org.apache.derby.impl.services.monitor.FileMonitor.startModule(Unknown Source)\n\tat org.apache.derby.iapi.services.monitor.Monitor.bootServiceModule(Unknown Source)\n\tat org.apache.derby.impl.store.raw.RawStore$6.run(Unknown Source)\n\tat java.security.AccessController.doPrivileged(Native Method)\n\tat org.apache.derby.impl.store.raw.RawStore.bootServiceModule(Unknown Source)\n\tat org.apache.derby.impl.store.raw.RawStore.boot(Unknown Source)\n\tat org.apache.derby.impl.services.monitor.BaseMonitor.boot(Unknown Source)\n\tat org.apache.derby.impl.services.monitor.TopService.bootModule(Unknown Source)\n\tat org.apache.derby.impl.services.monitor.BaseMonitor.startModule(Unknown Source)\n\tat org.apache.derby.impl.services.monitor.FileMonitor.startModule(Unknown Source)\n\tat org.apache.derby.iapi.services.monitor.Monitor.bootServiceModule(Unknown Source)\n\tat org.apache.derby.impl.store.access.RAMAccessManager$5.run(Unknown Source)\n\tat java.security.AccessController.doPrivileged(Native Method)\n\tat org.apache.derby.impl.store.access.RAMAccessManager.bootServiceModule(Unknown Source)\n\tat org.apache.derby.impl.store.access.RAMAccessManager.boot(Unknown Source)\n\tat org.apache.derby.impl.services.monitor.BaseMonitor.boot(Unknown Source)\n\tat org.apache.derby.impl.services.monitor.TopService.bootModule(Unknown Source)\n\tat org.apache.derby.impl.services.monitor.BaseMonitor.startModule(Unknown Source)\n\tat org.apache.derby.impl.services.monitor.FileMonitor.startModule(Unknown Source)\n\tat org.apache.derby.iapi.services.monitor.Monitor.bootServiceModule(Unknown Source)\n\tat org.apache.derby.impl.db.BasicDatabase$5.run(Unknown Source)\n\tat java.security.AccessController.doPrivileged(Native Method)\n\tat org.apache.derby.impl.db.BasicDatabase.bootServiceModule(Unknown Source)\n\tat org.apache.derby.impl.db.BasicDatabase.bootStore(Unknown Source)\n\tat org.apache.derby.impl.db.BasicDatabase.boot(Unknown Source)\n\tat org.apache.derby.impl.services.monitor.BaseMonitor.boot(Unknown Source)\n\tat org.apache.derby.impl.services.monitor.TopService.bootModule(Unknown Source)\n\tat org.apache.derby.impl.services.monitor.BaseMonitor.bootService(Unknown Source)\n\tat org.apache.derby.impl.services.monitor.BaseMonitor.startProviderService(Unknown Source)\n\tat org.apache.derby.impl.services.monitor.BaseMonitor.findProviderAndStartService(Unknown Source)\n\tat org.apache.derby.impl.services.monitor.BaseMonitor.startPersistentService(Unknown Source)\n\tat org.apache.derby.iapi.services.monitor.Monitor.startPersistentService(Unknown Source)\n\tat org.apache.derby.impl.jdbc.EmbedConnection$4.run(Unknown Source)\n\tat org.apache.derby.impl.jdbc.EmbedConnection$4.run(Unknown Source)\n\tat java.security.AccessController.doPrivileged(Native Method)\n\tat org.apache.derby.impl.jdbc.EmbedConnection.startPersistentService(Unknown Source)\n\t... 108 more\n------\r\n\n\tat sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)\n\tat sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)\n\tat sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)\n\tat java.lang.reflect.Constructor.newInstance(Constructor.java:423)\n\tat com.jolbox.bonecp.PoolUtil.generateSQLException(PoolUtil.java:192)\n\tat com.jolbox.bonecp.BoneCP.<init>(BoneCP.java:422)\n\tat com.jolbox.bonecp.BoneCPDataSource.getConnection(BoneCPDataSource.java:120)\n\tat org.datanucleus.store.rdbms.ConnectionFactoryImpl$ManagedConnectionImpl.getConnection(ConnectionFactoryImpl.java:501)\n\tat org.datanucleus.store.rdbms.RDBMSStoreManager.<init>(RDBMSStoreManager.java:298)\n\tat sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)\n\tat sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)\n\tat sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)\n\tat java.lang.reflect.Constructor.newInstance(Constructor.java:423)\n\tat org.datanucleus.plugin.NonManagedPluginRegistry.createExecutableExtension(NonManagedPluginRegistry.java:631)\n\tat org.datanucleus.plugin.PluginManager.createExecutableExtension(PluginManager.java:301)\n\tat org.datanucleus.NucleusContext.createStoreManagerForProperties(NucleusContext.java:1187)\n\tat org.datanucleus.NucleusContext.initialise(NucleusContext.java:356)\n\tat org.datanucleus.api.jdo.JDOPersistenceManagerFactory.freezeConfiguration(JDOPersistenceManagerFactory.java:775)\n\t... 83 more\nCaused by: java.sql.SQLException: Failed to start database 'metastore_db' with class loader org.apache.spark.sql.hive.client.IsolatedClientLoader$$anon$1@506f25ba, see the next exception for details.\n\tat org.apache.derby.impl.jdbc.SQLExceptionFactory.getSQLException(Unknown Source)\n\tat org.apache.derby.impl.jdbc.SQLExceptionFactory.getSQLException(Unknown Source)\n\tat org.apache.derby.impl.jdbc.Util.seeNextException(Unknown Source)\n\tat org.apache.derby.impl.jdbc.EmbedConnection.bootDatabase(Unknown Source)\n\tat org.apache.derby.impl.jdbc.EmbedConnection.<init>(Unknown Source)\n\tat org.apache.derby.jdbc.InternalDriver$1.run(Unknown Source)\n\tat org.apache.derby.jdbc.InternalDriver$1.run(Unknown Source)\n\tat java.security.AccessController.doPrivileged(Native Method)\n\tat org.apache.derby.jdbc.InternalDriver.getNewEmbedConnection(Unknown Source)\n\tat org.apache.derby.jdbc.InternalDriver.connect(Unknown Source)\n\tat org.apache.derby.jdbc.InternalDriver.connect(Unknown Source)\n\tat org.apache.derby.jdbc.AutoloadedDriver.connect(Unknown Source)\n\tat java.sql.DriverManager.getConnection(DriverManager.java:664)\n\tat java.sql.DriverManager.getConnection(DriverManager.java:208)\n\tat com.jolbox.bonecp.BoneCP.obtainRawInternalConnection(BoneCP.java:361)\n\tat com.jolbox.bonecp.BoneCP.<init>(BoneCP.java:416)\n\t... 95 more\nCaused by: ERROR XJ040: Failed to start database 'metastore_db' with class loader org.apache.spark.sql.hive.client.IsolatedClientLoader$$anon$1@506f25ba, see the next exception for details.\n\tat org.apache.derby.iapi.error.StandardException.newException(Unknown Source)\n\tat org.apache.derby.impl.jdbc.SQLExceptionFactory.wrapArgsForTransportAcrossDRDA(Unknown Source)\n\t... 111 more\nCaused by: ERROR XSDB6: Another instance of Derby may have already booted the database /Users/olga/yelper/metastore_db.\n\tat org.apache.derby.iapi.error.StandardException.newException(Unknown Source)\n\tat org.apache.derby.iapi.error.StandardException.newException(Unknown Source)\n\tat org.apache.derby.impl.store.raw.data.BaseDataFileFactory.privGetJBMSLockOnDB(Unknown Source)\n\tat org.apache.derby.impl.store.raw.data.BaseDataFileFactory.run(Unknown Source)\n\tat java.security.AccessController.doPrivileged(Native Method)\n\tat org.apache.derby.impl.store.raw.data.BaseDataFileFactory.getJBMSLockOnDB(Unknown Source)\n\tat org.apache.derby.impl.store.raw.data.BaseDataFileFactory.boot(Unknown Source)\n\tat org.apache.derby.impl.services.monitor.BaseMonitor.boot(Unknown Source)\n\tat org.apache.derby.impl.services.monitor.TopService.bootModule(Unknown Source)\n\tat org.apache.derby.impl.services.monitor.BaseMonitor.startModule(Unknown Source)\n\tat org.apache.derby.impl.services.monitor.FileMonitor.startModule(Unknown Source)\n\tat org.apache.derby.iapi.services.monitor.Monitor.bootServiceModule(Unknown Source)\n\tat org.apache.derby.impl.store.raw.RawStore$6.run(Unknown Source)\n\tat java.security.AccessController.doPrivileged(Native Method)\n\tat org.apache.derby.impl.store.raw.RawStore.bootServiceModule(Unknown Source)\n\tat org.apache.derby.impl.store.raw.RawStore.boot(Unknown Source)\n\tat org.apache.derby.impl.services.monitor.BaseMonitor.boot(Unknown Source)\n\tat org.apache.derby.impl.services.monitor.TopService.bootModule(Unknown Source)\n\tat org.apache.derby.impl.services.monitor.BaseMonitor.startModule(Unknown Source)\n\tat org.apache.derby.impl.services.monitor.FileMonitor.startModule(Unknown Source)\n\tat org.apache.derby.iapi.services.monitor.Monitor.bootServiceModule(Unknown Source)\n\tat org.apache.derby.impl.store.access.RAMAccessManager$5.run(Unknown Source)\n\tat java.security.AccessController.doPrivileged(Native Method)\n\tat org.apache.derby.impl.store.access.RAMAccessManager.bootServiceModule(Unknown Source)\n\tat org.apache.derby.impl.store.access.RAMAccessManager.boot(Unknown Source)\n\tat org.apache.derby.impl.services.monitor.BaseMonitor.boot(Unknown Source)\n\tat org.apache.derby.impl.services.monitor.TopService.bootModule(Unknown Source)\n\tat org.apache.derby.impl.services.monitor.BaseMonitor.startModule(Unknown Source)\n\tat org.apache.derby.impl.services.monitor.FileMonitor.startModule(Unknown Source)\n\tat org.apache.derby.iapi.services.monitor.Monitor.bootServiceModule(Unknown Source)\n\tat org.apache.derby.impl.db.BasicDatabase$5.run(Unknown Source)\n\tat java.security.AccessController.doPrivileged(Native Method)\n\tat org.apache.derby.impl.db.BasicDatabase.bootServiceModule(Unknown Source)\n\tat org.apache.derby.impl.db.BasicDatabase.bootStore(Unknown Source)\n\tat org.apache.derby.impl.db.BasicDatabase.boot(Unknown Source)\n\tat org.apache.derby.impl.services.monitor.BaseMonitor.boot(Unknown Source)\n\tat org.apache.derby.impl.services.monitor.TopService.bootModule(Unknown Source)\n\tat org.apache.derby.impl.services.monitor.BaseMonitor.bootService(Unknown Source)\n\tat org.apache.derby.impl.services.monitor.BaseMonitor.startProviderService(Unknown Source)\n\tat org.apache.derby.impl.services.monitor.BaseMonitor.findProviderAndStartService(Unknown Source)\n\tat org.apache.derby.impl.services.monitor.BaseMonitor.startPersistentService(Unknown Source)\n\tat org.apache.derby.iapi.services.monitor.Monitor.startPersistentService(Unknown Source)\n\tat org.apache.derby.impl.jdbc.EmbedConnection$4.run(Unknown Source)\n\tat org.apache.derby.impl.jdbc.EmbedConnection$4.run(Unknown Source)\n\tat java.security.AccessController.doPrivileged(Native Method)\n\tat org.apache.derby.impl.jdbc.EmbedConnection.startPersistentService(Unknown Source)\n\t... 108 more\n",
      "\nDuring handling of the above exception, another exception occurred:\n",
      "\u001b[0;31mAnalysisException\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-3-4d26e8c307c7>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      7\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      8\u001b[0m '''\n\u001b[0;32m----> 9\u001b[0;31m \u001b[0msf40\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mspark\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msql\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mselect\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     10\u001b[0m \u001b[0msf40\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshow\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m20\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/Cellar/apache-spark/2.3.2/libexec/python/pyspark/sql/session.py\u001b[0m in \u001b[0;36msql\u001b[0;34m(self, sqlQuery)\u001b[0m\n\u001b[1;32m    708\u001b[0m         \u001b[0;34m[\u001b[0m\u001b[0mRow\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mf1\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mf2\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34mu'row1'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mRow\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mf1\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m2\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mf2\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34mu'row2'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mRow\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mf1\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m3\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mf2\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34mu'row3'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    709\u001b[0m         \"\"\"\n\u001b[0;32m--> 710\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mDataFrame\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_jsparkSession\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msql\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msqlQuery\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_wrapped\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    711\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    712\u001b[0m     \u001b[0;34m@\u001b[0m\u001b[0msince\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m2.0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/Cellar/apache-spark/2.3.2/libexec/python/lib/py4j-0.10.7-src.zip/py4j/java_gateway.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args)\u001b[0m\n\u001b[1;32m   1255\u001b[0m         \u001b[0manswer\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgateway_client\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msend_command\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcommand\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1256\u001b[0m         return_value = get_return_value(\n\u001b[0;32m-> 1257\u001b[0;31m             answer, self.gateway_client, self.target_id, self.name)\n\u001b[0m\u001b[1;32m   1258\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1259\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mtemp_arg\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mtemp_args\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/Cellar/apache-spark/2.3.2/libexec/python/pyspark/sql/utils.py\u001b[0m in \u001b[0;36mdeco\u001b[0;34m(*a, **kw)\u001b[0m\n\u001b[1;32m     67\u001b[0m                                              e.java_exception.getStackTrace()))\n\u001b[1;32m     68\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0ms\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstartswith\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'org.apache.spark.sql.AnalysisException: '\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 69\u001b[0;31m                 \u001b[0;32mraise\u001b[0m \u001b[0mAnalysisException\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0ms\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msplit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m': '\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstackTrace\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     70\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0ms\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstartswith\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'org.apache.spark.sql.catalyst.analysis'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     71\u001b[0m                 \u001b[0;32mraise\u001b[0m \u001b[0mAnalysisException\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0ms\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msplit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m': '\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstackTrace\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mAnalysisException\u001b[0m: 'java.lang.RuntimeException: java.lang.RuntimeException: Unable to instantiate org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClient;'"
     ]
    }
   ],
   "source": [
    "select = '''\n",
    "SELECT  city\n",
    "FROM yelp_business \n",
    "WHERE review_count > 10  \n",
    "GROUP BY city\n",
    "\n",
    "\n",
    "'''\n",
    "sf40 = spark.sql(select)\n",
    "sf40.show(20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'sf40' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-2-a3ff6af05b2f>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mtype\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msf40\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m: name 'sf40' is not defined"
     ]
    }
   ],
   "source": [
    "type(sf40)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}