Digging Deep Into the New York Taxi Dataset

  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131

import pandas as pd
import numpy as np
import os
from itertools import islice
from io import StringIO
from tqdm.notebook import tqdm

pd.set_option("display.max_columns",None)
np.random.seed(42)


def reservoir_sample(iterable, random1, random2,random3,random4, k=1):
    
    '''
    From : https://en.wikipedia.org/wiki/Reservoir_sampling#An_optimal_algorithm
    
    '''
    
    iterator = iter(iterable)
    values = list(islice(iterator, k))

    W = np.exp(np.log(random1/k))
    while True:
        skip = int(np.floor(np.log(random2)/np.log(1-W)))
        selection = list(islice(iterator, skip, skip+1))
        if selection:
            values[random3] = selection[0]
            W *= np.exp(np.log(random4)/k)
        else:
            return values

def df_sample(filepath1,filepath2, k):
    
    r1,r2,r3,r4 = np.random.random(), np.random.random() ,np.random.randint(k), np.random.random()
    
    with open(filepath1, 'r') as f1, open(filepath2, 'r') as f2: 
        
        header1 = next(f1)
        header2 = next(f2)
        
        values1 = reservoir_sample(f1,r1,r2,r3,r4,k)  
        values2 = reservoir_sample(f2,r1,r2,r3,r4,k)  
        
        result1 = [header1] + values1
        result2 = [header2] + values2
        
    df1 = pd.read_csv(StringIO(''.join(result1)))
    df2 = pd.read_csv(StringIO(''.join(result2)))
    
    df1 = sample_preprocessing(df1)
    df2 = sample_preprocessing(df2, fare= True) #f2 has to be fare data
    
    return df1,df2


def sample_preprocessing(df_sample, fare= False):
    
    df_sample.columns = df_sample.columns.str.replace(" ","")
    
    if fare:
        df_sample.drop(columns=["hack_license","vendor_id"],inplace= True)  
    
    return df_sample


def reduce_memory(df, verbose=True):
    numerics = ["int8", "int16", "int32", "int64", "float16", "float32", "float64"]
    start_mem = df.memory_usage().sum() / 1024 ** 2
    for col in df.columns:
        col_type = df[col].dtypes
        if col_type in numerics:
            c_min = df[col].min()
            c_max = df[col].max()
            if str(col_type)[:3] == "int":
                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:
                    df[col] = df[col].astype(np.int8)
                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:
                    df[col] = df[col].astype(np.int16)
                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:
                    df[col] = df[col].astype(np.int32)
                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:
                    df[col] = df[col].astype(np.int64)
            else:
                if (
                    c_min > np.finfo(np.float16).min
                    and c_max < np.finfo(np.float16).max
                ):
                    df[col] = df[col].astype(np.float16)
                elif (
                    c_min > np.finfo(np.float32).min
                    and c_max < np.finfo(np.float32).max
                ):
                    df[col] = df[col].astype(np.float32)
                else:
                    df[col] = df[col].astype(np.float64)
    end_mem = df.memory_usage().sum() / 1024 ** 2
    if verbose:
        print(
            "Mem. usage decreased to {:.2f} Mb ({:.1f}% reduction)".format(
                end_mem, 100 * (start_mem - end_mem) / start_mem
            )
        )
    return df

data_file_list = sorted(os.listdir("../Data/NYC Trip Data/"), key = lambda x: int(x.split(".")[0].split("_")[-1]))
data_file_list = [os.path.join("../Data/NYC Trip Data/",file) for file in data_file_list]

fare_file_list = sorted(os.listdir("../Data/NYC Trip Fare/"), key = lambda x: int(x.split(".")[0].split("_")[-1]))
fare_file_list = [os.path.join("../Data/NYC Trip Fare/",file) for file in fare_file_list]

N = 150000 # Number of records per month chosen randomly 
df_data = []
df_fare = []
zipped_data = zip(data_file_list,fare_file_list)
for data_file, fare_file in tqdm(zipped_data, total = len(data_file_list)):
    
    df_data_sample, df_fare_sample = df_sample(data_file,fare_file,N)
    
    df_data.append(df_data_sample)
    df_fare.append(df_fare_sample)

fare = pd.concat(df_fare,axis=0,ignore_index=True)
data = pd.concat(df_data,axis=0,ignore_index=True)
data.drop_duplicates(subset=["medallion","pickup_datetime"],inplace=True)
fare.drop_duplicates(subset=["medallion","pickup_datetime"],inplace=True)

final_df = data.merge(fare, how="inner", on = ["medallion","pickup_datetime"])
final_df["pickup_datetime"] = pd.to_datetime(final_df["pickup_datetime"])
final_df["dropoff_datetime"] = pd.to_datetime(final_df["dropoff_datetime"])

final_df.to_csv("../Data/Pre-processed Data/data.csv")

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28

import warnings
warnings.filterwarnings("ignore")

import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import datetime 
import requests
import json
import os
from tqdm.notebook import tqdm
import geopy.distance

from shapely.geometry import Point
import folium
import geopandas as gpd
from geopandas import GeoDataFrame
import geoplot
import plotly.express as px
import plotly.graph_objects as go
from plotly.subplots import make_subplots

pd.set_option("display.max_columns",None)
np.random.seed(42)


df = pd.read_csv("../Data/Pre-processed Data/data.csv")
df.head()

1

df.info()

1

df.describe()

1

df.isnull().sum()

1
2
3

df["pickup_datetime"] = pd.to_datetime(df["pickup_datetime"])
df["dropoff_datetime"] = pd.to_datetime(df["dropoff_datetime"])
df.drop(df.columns[0], axis =1, inplace = True)

1
2
3
4
5

print(min(df.pickup_longitude.min(), df.dropoff_longitude.min()), \
max(df.pickup_longitude.max(), df.dropoff_longitude.max()))

print(min(df.pickup_latitude.min(), df.dropoff_latitude.min()), \
max(df.pickup_latitude.max(), df.dropoff_latitude.max()))

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13

NYBB = (-74.4505, -72.8478, 40.2734, 41.256) # https://boundingbox.klokantech.com/

before = df.shape[0]
print(f'Rows/Size Before: {before}')

df = df[(df.pickup_longitude >= NYBB[0]) & (df.pickup_longitude <= NYBB[1]) & \
           (df.pickup_latitude >= NYBB[2]) & (df.pickup_latitude <= NYBB[3]) & \
           (df.dropoff_longitude >= NYBB[0]) & (df.dropoff_longitude <= NYBB[1]) & \
           (df.dropoff_latitude >= NYBB[2]) & (df.dropoff_latitude <= NYBB[3])].reset_index(drop=True)

after = df.shape[0]
print(f'Rows/Size After: {after}')
print(f'Removed: {before - after}')

1

(df["dropoff_datetime"] - df["pickup_datetime"]).describe()

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16

before = df.shape[0]
print(f'Rows/Size Before: {before}')

invalid_trip_time_idx = np.where((df["dropoff_datetime"] - df["pickup_datetime"]).dt.total_seconds().astype(np.int64) != df["trip_time_in_secs"])[0]
trip_time_in_secs_ser = (df["dropoff_datetime"] - df["pickup_datetime"]).dt.total_seconds().astype(np.int64)
df.iloc[invalid_trip_time_idx, df.columns.tolist().index("trip_time_in_secs")] = trip_time_in_secs_ser.iloc[invalid_trip_time_idx]

invalid_duration_idx = np.where(((df["dropoff_datetime"] - df["pickup_datetime"]) == datetime.timedelta()) | ((df["dropoff_datetime"] - df["pickup_datetime"]) < datetime.timedelta()))[0]
df.drop(invalid_duration_idx, inplace=True)
df.reset_index(drop=True, inplace=True)

df.insert(loc=df.columns.tolist().index("trip_time_in_secs")+1,column="trip_time_in_min",value = (df["dropoff_datetime"] - df["pickup_datetime"]).astype("timedelta64[m]"))

after = df.shape[0]
print(f'Rows/Size After: {after}')
print(f'Removed: {before - after}')

1

(df["dropoff_datetime"] - df["pickup_datetime"]).describe()

1

trip_duration_percentiles = df["trip_time_in_secs"].quantile(np.round(np.arange(0.00, 1.01, 0.01), 2))

1

trip_duration_percentiles.loc[0.00:0.1]

1

trip_duration_percentiles.loc[0.90:]

1

df_short_duration.trip_distance.describe()

1
2
3
4
5
6
7
8
9

fig_short = px.scatter(df_short_duration, x="pickup_longitude",y="pickup_latitude")
fig_short = fig_short.update_traces(marker=dict(size= 1),name="Short Duration Pickups")
fig_short.data[0].showlegend = True
fig_short = fig_short.add_trace(go.Scatter(x=df_short_duration["dropoff_longitude"], 
                         y=df_short_duration["dropoff_latitude"], mode='markers', marker=dict(color="red", size=1),
                                          name="Short Duration Dropoffs", opacity=0.5))
fig_short = fig_short.update_layout(xaxis_title ="Longitude", yaxis_title = "Latitude")

fig_short.show()

1

df_large_duration = df[(df.trip_time_in_secs >= 43200)] # 12 hours max, See : https://www1.nyc.gov/assets/tlc/downloads/pdf/rule_book_current_chapter_58.pdf#page=34

1

df_large_duration.trip_distance.describe()

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11

before = df.shape[0]
print(f'Rows/Size Before: {before}')

invalid_duration_idx = df[(df.trip_time_in_secs.between(1,5)) | (df.trip_time_in_secs >= 43200)].index.tolist()

df.drop(invalid_duration_idx, inplace=True)
df.reset_index(drop=True, inplace= True)

after = df.shape[0]
print(f'Rows/Size After: {after}')
print(f'Removed: {before - after}')

1

df.trip_distance.describe().apply(lambda x: '%.5f' % x)

1

trip_distance_percentiles = df["trip_distance"].quantile(np.round(np.arange(0.00, 1.01, 0.01), 2))

1

trip_distance_percentiles.loc[0.00:0.1]

1

trip_distance_percentiles.loc[0.90:]

1

px.histogram(df.sample(500000), x= "trip_distance") #obvious outliers (ran several times due to sampling)

1

df.trip_distance.where(lambda x: x > df.trip_distance.min()).min() # Second smallest distance is 0.01 miles or 16 meters

1

df.trip_distance.nsmallest(500000).unique()

1

df.trip_distance.nlargest(10)

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11

before = df.shape[0]
print(f'Rows/Size Before: {before}')

invalid_distance_idx = df[(df.trip_distance > 100) | (df.trip_distance == 0)].index.tolist() # Remove distance of 0 and greater than 100 miles https://www.google.com/maps/dir/40.5055432,-74.2381798/41.0191685,-73.7871866/@40.9358817,-74.3487811,8.92z/data=!3m1!5s0x89c2f2be234865c9:0x12533050f3f45b3c!4m2!4m1!3e0

df.drop(invalid_distance_idx, inplace=True)
df.reset_index(drop=True, inplace= True)

after = df.shape[0]
print(f'Rows/Size After: {after}')
print(f'Removed: {before - after}')

1
2

df.insert(loc=df.columns.tolist().index("trip_distance")+1,column="trip_distance_km",value = np.round(df["trip_distance"] * 1.609344,3))
df.insert(loc=df.columns.tolist().index("trip_distance_km")+1,column="trip_speed_mi/hr",value = np.round((df["trip_distance"]/df["trip_time_in_secs"])*60*60,3))

1

df["trip_speed_mi/hr"].describe().apply(lambda x: '%.5f' % x)

1

speed_percentiles = df["trip_speed_mi/hr"].quantile(np.round(np.arange(0.00, 1.01, 0.01), 2))

1

speed_percentiles.loc[0:0.1]

1

speed_percentiles.loc[0.90:] # very odd maxmimum speed

1

df["trip_speed_mi/hr"].nlargest(30)

1
2
3
4
5
6
7

speeds = df["trip_speed_mi/hr"].values
speeds = np.sort(speeds,axis = None)

for i in np.arange(0.0, 1.0, 0.1):
    print("{} percentile is {}".format(99+i, speeds[int(len(speeds) * (float(99+i) / 100) )]))
print("100 percentile value is ",speeds[-1])
del speeds

1
2
3
4
5
6
7
8

before = df.shape[0]
print(f'Rows/Size Before: {before}')

df = df[(df["trip_speed_mi/hr"] > 0) & (df["trip_speed_mi/hr"] <= 50)].reset_index(drop=True) # 99.9th percentile is 48.027

after = df.shape[0]
print(f'Rows/Size After: {after}')
print(f'Removed: {before - after}')

1

df["total_amount"].describe().apply(lambda x: '%.5f' % x)

1

df["fare_amount"].describe().apply(lambda x: '%.5f' % x)

1

df["surcharge"].describe().apply(lambda x: '%.5f' % x)

1

df["mta_tax"].describe().apply(lambda x: '%.5f' % x)

1

df["tip_amount"].describe().apply(lambda x: '%.5f' % x)

1

df["tolls_amount"].describe().apply(lambda x: '%.5f' % x)

1
2

total_amount_percentiles = df["total_amount"].quantile(np.round(np.arange(0.00, 1.01, 0.01), 2))
total_amount_percentiles.loc[0:0.1], total_amount_percentiles.loc[0.90:]

1

df.total_amount.nsmallest(20)

1

df.total_amount.nlargest(20)

1
2
3
4
5
6
7

totals = df["total_amount"].values
totals = np.sort(totals,axis = None)

for i in np.arange(0.0, 1.0, 0.1):
    print("{} percentile is {}".format(99+i, totals[int(len(totals) * (float(99+i) / 100) )]))
print("100 percentile value is ",totals[-1])
del totals

1
2
3
4
5
6
7
8

before = df.shape[0]
print(f'Rows/Size Before: {before}')

df = df[(df["total_amount"] >= 0) & (df["total_amount"] <= 150)].reset_index(drop=True)   # See: https://www.introducingnewyork.com/taxis

after = df.shape[0]
print(f'Rows/Size After: {after}')
print(f'Removed: {before - after}')

1
2

fare_amount_percentiles = df["fare_amount"].quantile(np.round(np.arange(0.00, 1.01, 0.01), 2))
fare_amount_percentiles.loc[0:0.1], fare_amount_percentiles.loc[0.90:]

1
2
3
4
5
6
7
8

before = df.shape[0]
print(f'Rows/Size Before: {before}')

df = df[(df["fare_amount"] >= 0)].reset_index(drop=True)

after = df.shape[0]
print(f'Rows/Size After: {after}')
print(f'Removed: {before - after}')

1
2

surcharge_percentiles = df["surcharge"].quantile(np.round(np.arange(0.00, 1.01, 0.01), 2))
surcharge_percentiles.loc[0:0.1], surcharge_percentiles.loc[0.9:]

1

df.surcharge.nlargest(10)

1
2

mta_percentiles = df["mta_tax"].quantile(np.round(np.arange(0.00, 1.01, 0.01), 2))
mta_percentiles.loc[0:0.1], mta_percentiles.loc[0.9:]

1
2

tip_amount_percentiles = df["tip_amount"].quantile(np.round(np.arange(0.00, 1.01, 0.01), 2))
tip_amount_percentiles.loc[0.0:0.1], tip_amount_percentiles.loc[0.9:]

1

df.tip_amount.nlargest(20)

1
2

tolls_percentiles = df["tolls_amount"].quantile(np.round(np.arange(0.00, 1.01, 0.01),2 ))
tolls_percentiles.loc[0.0:0.1], tolls_percentiles.loc[0.9:]

1

df.tolls_amount.nlargest(10)

1

df["passenger_count"].describe().apply(lambda x: '%.5f' % x) # Looks clean but we have passenger counts of 0

1
2
3

print(df[df['passenger_count'] == 0].shape[0]) #7 trips with passenger count 0, replace with mean value

df.loc[df[df['passenger_count'] == 0].index,'passenger_count'] = df.passenger_count.mean()

1

df.passenger_count.quantile(np.round(np.arange(0.00,1.01, 0.01),2)).loc[0.9:]

1

df.select_dtypes(exclude=["number","datetime"]).columns #rate code is missing as its numerical, but will be treated as categorical

1

df.rate_code.unique() # Should only be 1,2,3,4,5,6 from data description, See: https://www1.nyc.gov/assets/tlc/downloads/pdf/data_dictionary_trip_records_yellow.pdf

1
2
3
4

allowed = [1,2,3,4,5,6]

abnormal_rate_code = df[~df.rate_code.isin(allowed)]
print(abnormal_rate_code.shape[0])

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17

newyork_center= folium.Map(location=[40.7345, -73.8697]) # https://www.google.com/search?q=middle+of+new+york+coordinates&rlz=1C1CHBF_enDE1000DE1000&oq=middle+of+new+york+coordina&aqs=chrome.1.69i57j33i22i29i30l6.7294j0j7&sourceid=chrome&ie=UTF-8

for pickup_lat,pick_long, dropoff_lat, dropoff_long, code in zip(abnormal_rate_code["pickup_latitude"],
                                                                 abnormal_rate_code["pickup_longitude"],
                                                                 abnormal_rate_code["dropoff_latitude"],
                                                                 abnormal_rate_code["dropoff_longitude"],
                                                                 abnormal_rate_code["rate_code"][0:14]):
    
    folium.CircleMarker([pickup_lat, pick_long],popup=code,tooltip = "Pickup",fill_color= "#d1005b",color="#d1005b",
                       fill = True,radius = 5).add_to(newyork_center)
                  
    folium.CircleMarker([dropoff_lat, dropoff_long],popup=code,tooltip = "Dropoff",fill_color= "#616161",color="#616161",
                       fill = True,radius = 5).add_to(newyork_center)
    
    folium.PolyLine([[pickup_lat, pick_long],[dropoff_lat,dropoff_long]]).add_to(newyork_center)
    
newyork_center 

1
2

abnormal_rate_code_idx = abnormal_rate_code.index.tolist()
df.loc[abnormal_rate_code_idx, 'rate_code'] = 1

1

df.payment_type.unique() # Looks fine CSH = Cash, DIS = Dispute, CRD = Credit Card, UNK = Unknown, NOC = No charge

1

df.store_and_fwd_flag.unique() # Looks fine Y= store and forward trip, N= not a store and forward trip

1

df.vendor_id.unique() # Looks fine CMT = Creative Mobile Technologies, VTS = VeriFone Inc.

1

df.to_csv("../Data/Pre-processed Data/cleaned_data.csv",index=False)

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44

import warnings
warnings.filterwarnings("ignore")

import pandas as pd
import numpy as np
import scipy.stats as ss
from statsmodels.distributions.empirical_distribution import ECDF

import matplotlib.pyplot as plt
import seaborn as sns
import datetime 
import requests
import json
import os
from tqdm.notebook import tqdm
from tabulate import tabulate
import geopy.distance

from sklearn.model_selection import train_test_split

from shapely.geometry import Point,Polygon, MultiPoint,MultiPolygon
import folium
from folium.plugins import HeatMapWithTime
from geopy.geocoders import Nominatim
geolocator = Nominatim(user_agent="taxi-analyis")
import geopandas as gpd
from geopandas import GeoDataFrame
import geoplot
import pygeohash as pgh
import pyproj
from area import area

import plotly.express as px
import plotly.graph_objects as go
from plotly.subplots import make_subplots
import plotly.offline as pyo
pyo.init_notebook_mode(connected=True)

from PIL import Image

pd.set_option("display.max_columns",None)
np.random.seed(42)

df = pd.read_csv("../Data/Pre-processed Data/cleaned_data.csv")

1
2
3
4
5
6
7
8

df['pickup_datetime'] = pd.to_datetime(df['pickup_datetime'])
df['dropoff_datetime'] = pd.to_datetime(df['dropoff_datetime'])


# Too many addresses to geocode, Nominatim usage policy doesn't allow bulk geocoding and will timeout (was going to be used in hover)

#df["pickup_address"] = df.apply(lambda x: geolocator.reverse(str(x.pickup_latitude) + "," + str(x.pickup_longitude),timeout=5)[0],axis=1)
#df["dropoff_address"] = df.apply(lambda x: geolocator.reverse(str(x.dropoff_latitude) + "," + str(x.dropoff_longitude),timeout=5)[0],axis=1)

1
2
3
4
5

df_train, df_test = train_test_split(df, test_size=0.33,random_state=32)
df_train.reset_index(drop=True, inplace=True)
df_test.reset_index(drop=True, inplace=True)

df_train.shape, df_test.shape

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80

def getBB(df, BB):

    df_BB = df[(df.pickup_longitude >= BB[0]) & (df.pickup_longitude <= BB[1]) & 
               (df.pickup_latitude >= BB[2]) & (df.pickup_latitude <= BB[3]) &
               (df.dropoff_longitude >= BB[0]) & (df.dropoff_longitude <= BB[1]) &
               (df.dropoff_latitude >= BB[2]) & (df.dropoff_latitude <= BB[3])].reset_index(drop=True)
    
    return df_BB

def drop_pick_map(df, BB, sample_size = 1000, title = '' ,size=5, opacity=0.8):
    
    
    df_plt = df.sample(sample_size)
    df_plt = getBB(df_plt, BB)
    
    data = [go.Scattermapbox(
            lat= df_plt['pickup_latitude'] ,
            lon= df_plt['pickup_longitude'],
            text = ['Trip Distance (mi): '+ str(df_plt['trip_distance'][i]) + "<br>" + 'Trip Speed (mi/hr): ' + str(df_plt['trip_speed_mi/hr'][i]) for i in range(df_plt.shape[0])],
            hovertext=df_plt['trip_time_in_secs'],
            customdata = df_plt['total_amount'],
            mode='markers',
            marker=dict(
                size= size,
                color = 'red',
                opacity = opacity),
            name ='Pickups',
            subplot='mapbox',
            hovertemplate = "Latitude (°): %{lat} <br>Longitude (°): %{lon} <br>Trip Duration (s): %{hovertext} <br>%{text} <br>Trip Amount ($): %{customdata}"
          ),
        go.Scattermapbox(
            lat= df_plt['dropoff_latitude'] ,
            lon= df_plt['dropoff_longitude'],
            text = ['Trip Distance (mi): '+ str(df_plt['trip_distance'][i]) + "<br>" + 'Trip Speed (mi/hr): ' + str(df_plt['trip_speed_mi/hr'][i]) for i in range(df_plt.shape[0])],
            hovertext=df_plt['trip_time_in_secs'],
            customdata = df_plt['total_amount'],
            mode='markers',
            marker=dict(
                size= size,
                color = 'blue',
                opacity = opacity),
            name ='Dropoffs',
            subplot ='mapbox2',
            hovertemplate = "Latitude (°): %{lat} <br>Longitude (°): %{lon} <br>Trip Duration (s): %{hovertext} <br>%{text} <br>Trip Amount ($): %{customdata}"
          )]

    layout = go.Layout(autosize=False,
                       mapbox= dict(center= dict(
                                             lat=40.721319,
                                             lon=-73.987130),
                                   style = 'open-street-map',
                                   zoom = 8,
                                   domain={'x': [0, 0.5], 'y': [0, 1]}),
                       mapbox2 = dict(center= dict(
                                             lat=40.721319,
                                             lon=-73.987130),
                                   style = 'open-street-map',
                                   zoom=8,
                                   domain={'x': [0.51, 1.0], 'y': [0, 1]}),
                        width=1000,
                        height=600, title = title)

    fig = go.Figure(data=data, layout=layout)
    
    
    fig.show()
    
    return 

#All hires computationaly expensive, can't use interactive plots opting from matplotlib
def all_hires(df, BB, figsize=(12, 12), ax=None, c=('r', 'b')):
    if ax == None:
        fig, ax = plt.subplots(1, 1, figsize=figsize)

    df_sub = getBB(df,BB)
    ax.scatter(df_sub.pickup_longitude, df_sub.pickup_latitude, c=c[0], s=0.1, alpha=0.5)
    ax.scatter(df_sub.dropoff_longitude, df_sub.dropoff_latitude, c=c[1], s=0.1, alpha=0.5)
    plt.legend(["Pickups","Dropoffs"])

NYBB = (-74.15, -73.7004, 40.5774, 40.9176)

1

drop_pick_map(df_train, NYBB, title = 'Map of Pickups & Dropoffs',sample_size=10000)

1

all_hires(df_train,NYBB)

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12

columns=['trip_distance','trip_speed_mi/hr','trip_time_in_secs']
correlation_amount = pd.DataFrame(index=["total_amount"],columns=columns)
for i,col in enumerate(columns):
    cor = df_train.total_amount.corr(df_train[col])
    correlation_amount.loc[:,columns[i]] = cor
plt.figure(figsize=(25,5))
g = sns.heatmap(correlation_amount,annot=True,fmt='.4f', annot_kws={"fontsize":16})
g.set_yticklabels(['Tip Amount'], fontsize = 16)
g.set_xticklabels(['Trip Distance','Trip Speed', 'Trip Duration'], fontsize = 16)
g.set_yticklabels(['Total Amount'], fontsize = 16)
plt.title("Correlation Matrix between Total Amount and Numerical Features", fontdict={'fontsize':20})
plt.plot()

1
2
3
4
5

df_30_less = df_train[(df_train['trip_distance_km'] <= 30)].reset_index(drop=True)
df_50_less = df_train[(df_train['trip_distance_km'] <= 50)].reset_index(drop=True)

#more severe
df_5_less = df_train[(df_train['trip_distance_km'] <= 5)].reset_index(drop=True)

1
2
3
4

print("Mean Total Amount for 30 km or less: ", df_30_less.total_amount.mean())
print("Mean Total Amount for 50 km or less: ", df_50_less.total_amount.mean())
print("Mean Total Amount for 5 km or less: ", df_5_less.total_amount.mean())
print("Mean Total Amount for no distance restriction: ", df_train.total_amount.mean())

 1
 2
 3
 4
 5
 6
 7
 8
 9
10

fig = go.Figure()

fig.add_trace(go.Box(y=df_30_less["total_amount"], name= "30 km or less earnings distribution"))
fig.add_trace(go.Box(y=df_50_less["total_amount"], name= "50 km or less earnings distribution"))
fig.add_trace(go.Box(y=df_5_less["total_amount"], name= "5 km or less earnings distribution"))

fig.add_trace(go.Box(y=df_train["total_amount"],name= "No distance limit earnings distribution"))

fig = fig.update_layout(title = "T1: How much do taxis earn per trip according to distance restrictions")
fig.write_html("../Figures/T1/T1_fig.html")

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11

all_log = np.log(df_train.total_amount.values)
km30_log = np.log(df_30_less.total_amount.values)
km50_log = np.log(df_50_less.total_amount.values)

fig = plt.figure(figsize=(15,8))
plt.title("Log Distribution of Total Amount")
plt.hist(all_log[np.isfinite(all_log)],bins=60, label='No restriction', alpha = 0.5)
plt.hist(km30_log[np.isfinite(km30_log)],bins=65,label='30 km or less', alpha=0.5)
plt.hist(km50_log[np.isfinite(km50_log)],bins=70, label = '50 km or less',alpha=0.5)
plt.legend(loc='upper right')
plt.show()

1
2
3
4
5
6
7

fig = plt.figure(figsize=(15,8))
plt.title("Distribution of Total Amount")
plt.hist(df_train.total_amount.values,bins=60, label='No restriction', alpha = 0.5)
plt.hist(df_30_less.total_amount.values,bins=65,label='30 km or less', alpha=0.5)
plt.hist(df_50_less.total_amount.values,bins=70, label = '50 km or less',alpha=0.5)
plt.legend(loc='upper right')
plt.show()

1
2
3
4
5
6
7
8

ks_30vs_all = ss.ks_2samp(df_train['total_amount'].values, df_30_less['total_amount'].values)
ks_50vs_all = ss.ks_2samp(df_train['total_amount'].values, df_50_less['total_amount'].values)
ks_5vs_all = ss.ks_2samp(df_train['total_amount'].values, df_5_less['total_amount'].values)


print(f"KS-Statistics between 30 km restriction and no restriction: {ks_30vs_all.statistic}, p-value: {ks_30vs_all.pvalue}")
print(f"KS-Statistics between 50 km restriction and no restriction: {ks_50vs_all.statistic}, p-value: {ks_50vs_all.pvalue}")
print(f"KS-Statistics between 5 km restriction and no restriction: {ks_5vs_all.statistic}, p-value: {ks_5vs_all.pvalue}")

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14

ecdf_norestrict = ECDF(df_train.total_amount.values)
ecdf_30 = ECDF(df_30_less.total_amount.values)
ecdf_50 = ECDF(df_50_less.total_amount.values)
ecdf_5 = ECDF(df_5_less.total_amount.values)

fig = plt.figure(figsize=(15,8))
plt.plot(ecdf_norestrict.x, ecdf_norestrict.y, label='No Restriction', alpha=0.5)
plt.plot(ecdf_30.x, ecdf_30.y, label = '30 km or less', alpha=0.5)
plt.plot(ecdf_50.x, ecdf_50.y, label = '50 km or less',alpha=0.5)
plt.plot(ecdf_5.x, ecdf_5.y, label = '5 km or less',alpha=0.5)
_= plt.xlabel('Total Amount',fontdict=dict(size=14))
_= plt.ylabel('Probability',fontdict=dict(size=14))
plt.legend()
plt.show()