# Import necessary libraries
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
from sklearn.model_selection import train_test_split
from sklearn.linear_model import LinearRegression
from sklearn.cluster import KMeans

# Save data to the DataFrame

# List to store the dataframes
dfs = []
base_path = "/content/drive/MyDrive/Colab Notebooks/CMSC320/Final Tutorial/Crime_Incidents_in_"

# Read the CSV files
for year in range(2008, 2023):
    file_path = base_path + str(year) + ".csv"
    crime_data = pd.read_csv(file_path)
    crime_data['YEAR'] = year # Add a column for the year
    dfs.append(crime_data)

# Given an example of what the data looks like
dfs[0]

# Create a new DataFrame that combine all dataframes and removes unneeded data
df = pd.concat(dfs, axis=0, ignore_index=True)
df = df.drop(['X', 'Y', 'CCN', 'BLOCK', 'XBLOCK', 'YBLOCK', 'BLOCK_GROUP', 'BID', 'METHOD', 'YEAR',
              'OBJECTID', 'WARD', 'ANC', 'DISTRICT', 'PSA', 'OCTO_RECORD_ID', 'NEIGHBORHOOD_CLUSTER'], axis = 1)

# Modify the dataframe, to extract needed data and delete unneeded data further
df['REPORT_DAT'] = pd.to_datetime(df['REPORT_DAT'])
df['START_DATE'] = pd.to_datetime(df['START_DATE'])
df['REPORT_YEAR'] = df['REPORT_DAT'].dt.year
df['START_TIME'] = df['START_DATE'].dt.hour
df['WEEKDAY'] = df['START_DATE'].dt.day_name()

df['LOCATION'] = np.array(df[['LATITUDE','LONGITUDE']]).tolist()
df = df.drop(['LATITUDE', 'LONGITUDE', 'CENSUS_TRACT'], axis = 1)

# Delete duplicate values
df.drop_duplicates(subset=df.columns.difference(['LOCATION']))

# Delete reports that not in the year of 2008 to 2022
years_to_keep = range(2008, 2023)
df = df[df['REPORT_YEAR'].isin(years_to_keep)]

# Delete reports that not have occurred in the report year
df_cleaned = df[df['REPORT_YEAR'] == df['START_DATE'].dt.year]

# Sort the data based on the start data time
df = df_cleaned.sort_values(by='START_DATE')
df = df.reindex(columns=['REPORT_YEAR',  'REPORT_DAT', 'START_DATE', 'START_TIME', 'END_DATE', 'WEEKDAY', 'SHIFT', 'OFFENSE', 'LOCATION', 'VOTING_PRECINCT'])
df

# Line Graph - Crime Count by Year
crime_count = df.groupby('REPORT_YEAR').size()

plt.figure(figsize=(10, 5))
plt.plot(crime_count.index, crime_count.values, marker='o')
plt.title('Crime Count by Year')
plt.xlabel('Year')
plt.ylabel('Crime Count')
plt.grid(True)
plt.show()

# Bar chart of crime types by year
# We are considerig the Covid time for this period
crime_count = df.groupby(['OFFENSE', 'REPORT_YEAR']).size().unstack()
crime_count = crime_count.loc[:, 2020:2022]

# Create a bar chart
plt.figure(figsize=(10, 6))
crime_count.plot(kind='bar', stacked=True)
plt.title('Crime Types by Year(2020-2022)')
plt.xlabel('Year')
plt.ylabel('Crime Count')
plt.legend(title='Crime Type')
plt.show()

# Crime Count by Shift
shift_counts = df['SHIFT'].value_counts()

# Bar plot for crime count by shift
plt.figure(figsize=(10, 6))
plt.bar(shift_counts.index, shift_counts.values)
plt.xlabel('Shift')
plt.ylabel('Crime Count')
plt.title('Crime Count by Shift')
plt.show()

# Crime Count by Offense Type
crime_count_offense = df['OFFENSE'].value_counts().head(8)

plt.figure(figsize=(12, 6))

# Pie Chart
# Part of the code for plotting this image are from stack overflow
plt.subplot(121)
plt.pie(crime_count_offense, labels=crime_count_offense.index, autopct='%1.1f%%')
plt.title('Crime Count by Offense Type (Top 8)')
plt.axis('equal')

# Bar Plot
# Part of the code for plotting this image are from stack overflow
plt.subplot(122)
plt.barh(crime_count_offense.index[::-1], crime_count_offense.values[::-1])
plt.title('Crime Count by Offense Type (Top 8)')
plt.xlabel('Count')

plt.tight_layout()
plt.show()

# Top 30 Voting Precincts by Crime Count
# Group the data by voting precinct and calculate the crime count for each precinct
precinct_counts = df['VOTING_PRECINCT'].value_counts().sort_values(ascending=False)

# Select the top 10 precincts with the highest crime count
top_precincts = precinct_counts.head(30)

# Bar plot for crime count by voting precinct
plt.figure(figsize=(12, 6))
plt.bar(top_precincts.index, top_precincts.values)
plt.xlabel('Voting Precinct')
plt.ylabel('Crime Count')
plt.title('Top 30 Voting Precincts by Crime Count')
plt.xticks(rotation=90)  # Rotate x-axis labels for better readability
plt.tight_layout()
plt.show()

# Crime Rates by Day of the Week
# Considering the influence of COVID, we seperate the data to 2008-2019 and 2020-2022
df_normal = df[df['REPORT_YEAR'] <= 2019]
df_covid = df[(df['REPORT_YEAR'] >= 2020) & (df['REPORT_YEAR'] <= 2022)]


# Calculate the crime rates by day of the week
weekday_counts_n = df_normal['WEEKDAY'].value_counts()
crime_rates_n = (weekday_counts_n / weekday_counts_n.sum()) * 100

weekday_counts_c = df_covid['WEEKDAY'].value_counts()
crime_rates_c = (weekday_counts_c / weekday_counts_c.sum()) * 100

# Reindex the crime rates by day of the week based on the desired order
crime_rates_n = crime_rates_n.reindex(['Monday', 'Tuesday', 'Wednesday', 
                                                 'Thursday', 'Friday', 'Saturday', 'Sunday'])
crime_rates_c = crime_rates_c.reindex(['Monday', 'Tuesday', 'Wednesday', 
                                                 'Thursday', 'Friday', 'Saturday', 'Sunday'])

# Create a line graph
plt.figure(figsize=(16, 5))
plt.subplot(121)
plt.plot(crime_rates_n.index, crime_rates_n.values, marker='o')
plt.title('Crime Rates by Day of the Week(2008-2019)')
plt.xlabel('Day of the Week')
plt.ylabel('Crime Rate (%)')
plt.grid(True)

plt.subplot(122)
plt.plot(crime_rates_c.index, crime_rates_c.values, marker='o')
plt.title('Crime Rates by Day of the Week(2020-2022)')
plt.xlabel('Day of the Week')
plt.ylabel('Crime Rate (%)')
plt.grid(True)

plt.show()

# Heapmap about crime count by year and weekday
# Calculate the crime count for each year and weekday
crime_count = df.groupby(['REPORT_YEAR', 'WEEKDAY']).size().reset_index(name='COUNT')

# Pivot the data to create a pivot table
pivot_table = crime_count.pivot(index='WEEKDAY', columns='REPORT_YEAR', values='COUNT')

# Set the order of weekdays for proper sorting on the y-axis
weekday_order = ['Monday', 'Tuesday', 'Wednesday', 'Thursday', 'Friday', 'Saturday', 'Sunday']

# Plot the heatmap
# Part of the code for plotting this image are from stack overflow
plt.figure(figsize=(10, 6))
plt.imshow(pivot_table.values, cmap='YlOrRd', aspect='auto')
plt.xticks(range(len(pivot_table.columns)), pivot_table.columns)
plt.yticks(range(len(pivot_table.index)), weekday_order)
plt.colorbar(label='Crime Count')
plt.title('Crime Count by Year and Weekday')
plt.xlabel('Year')
plt.ylabel('Weekday')
plt.show()

# Part of the code for plotting this image are from stack overflow

plt.figure(figsize=(12,8))
# Specify the order of offense categories
offense_order = ['ROBBERY', 'THEFT F/AUTO', 'THEFT/OTHER', 'MOTOR VEHICLE THEFT',
                 'ASSAULT W/DANGEROUS WEAPON', 'BURGLARY', 'SEX ABUSE', 'HOMICIDE', 'ARSON']

# Create a FacetGrid with separate violin plots for each year
g = sns.FacetGrid(df, col='REPORT_YEAR', col_wrap=3)
g.map(sns.violinplot, 'START_TIME', 'OFFENSE', order=offense_order)

# Set the titles and labels
g.set_titles("Violin Plot of Offense \n by Time - Year {col_name}")
g.set_xlabels('Start Time')
g.set_ylabels('Offense')

# Adjust the spacing between subplots
plt.tight_layout()

# Display the plot
plt.show()

# Predict the crime count everyday in the future using regression
# Some of the code are from stack overflow

df_ml = df.copy()
# Convert 'START_DATE' to the Day period
df_ml['START_DATE'] = df_ml['START_DATE'].dt.date

# We normalize the date in order to do the regression
df_ml['START_DATE'] = (df_ml['START_DATE'] - df_ml['START_DATE'].min())  / np.timedelta64(1,'D')

# Calculate the crime count everyday
df_ml['CRIME_COUNT'] = df_ml.groupby('START_DATE')['REPORT_DAT'].transform('count')

# Drop the unnecessary columns
df_ml = df_ml.drop(['REPORT_YEAR','REPORT_DAT', 'START_TIME', 'END_DATE', 'WEEKDAY', 'SHIFT', 'OFFENSE', 'LOCATION', 'VOTING_PRECINCT'], axis=1)
df_ml.drop_duplicates(subset=['START_DATE', 'CRIME_COUNT'], keep='first', inplace=True)

df_ml

# Initialize a linear regression model
model = LinearRegression()

X = df_ml['START_DATE'].values.reshape(-1, 1)
y = df_ml['CRIME_COUNT'].values

# Train the model
model.fit(X, y)

# Generate a range of dates for prediction
dates = np.arange(df_ml['START_DATE'].min(), df_ml['START_DATE'].max()+1).reshape(-1, 1)

# Make predictions using the trained model
predictions = model.predict(dates)

# Plot the original data and the regression line
plt.scatter(df_ml['START_DATE'], df_ml['CRIME_COUNT'], color='b', label='Original Data')
plt.plot(dates, predictions, color='r', label='Regression Line')
plt.xlabel('Start Date')
plt.ylabel('Crime Count')
plt.title('Linear Regression')
plt.legend()
plt.show()

# Extract the coordinate
cords = df['LOCATION']
lat = [cord[0] for cord in cords]
long = [cord[1] for cord in cords]

# Perform K-means clustering on lat-long data
X = pd.DataFrame({'latitude': lat, 'longitude': long}).values

# Determine the optimal number of clusters using the elbow method
# Some code are generated by ChatGPT
inertia = []
k_range = range(1, 30)  # Test different numbers of clusters
for k in k_range:
    kmeans = KMeans(n_clusters=k, n_init=10, random_state=42)
    kmeans.fit(X)
    inertia.append(kmeans.inertia_)

# Plot the elbow curve to find the optimal number of clusters
plt.plot(k_range, inertia, 'bo-')
plt.xlabel('Number of Clusters (k)')
plt.ylabel('Within-cluster Sum of Squares (Inertia)')
plt.title('Elbow Curve')
plt.show()

# Choose the optimal number of clusters based on the elbow curve
k_optimal = 3  # Adjust the value based on the elbow curve

# Perform K-means clustering with the optimal number of clusters
kmeans = KMeans(n_clusters=k_optimal,  n_init=10, random_state=42)
kmeans.fit(X)
labels = kmeans.labels_
centroids = kmeans.cluster_centers_

# Plot the crime clusters
plt.scatter(X[:, 1], X[:, 0], c=labels, cmap='viridis')
plt.scatter(centroids[:, 1], centroids[:, 0], c='red', marker='x')
plt.xlabel('Longitude')
plt.ylabel('Latitude')
plt.title('Crime Clusters')
plt.show()