{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "# import pm4py\n",
    "from tqdm import tqdm\n",
    "import os\n",
    "import numpy as np\n",
    "import math"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Importing data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "712\n",
      "3001\n"
     ]
    }
   ],
   "source": [
    "# declined_train = pd.read_csv('../data/declined_train.csv')\n",
    "declined_test = pd.read_csv('../data/declined_test.csv', index_col=0).reset_index(drop=True)\n",
    "print(len(declined_test['case:concept:name'].unique()))\n",
    "\n",
    "declined_train = pd.read_csv('../data/declined_train.csv', index_col=0).reset_index(drop=True)\n",
    "print(len(declined_train['case:concept:name'].unique()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "from random import sample\n",
    "approved_train = pd.read_csv('../data/approved_train.csv', index_col=0).reset_index(drop=True)\n",
    "approved_train = approved_train[approved_train['case:concept:name'].isin(sample(list(approved_train['case:concept:name'].unique()), 3000))]\n",
    "\n",
    "approved_test = pd.read_csv('../data/approved_test.csv', index_col=0).reset_index(drop=True)\n",
    "approved_test = approved_test[approved_test['case:concept:name'].isin(sample(list(approved_test['case:concept:name'].unique()), 700))]\n",
    "\n",
    "\n",
    "\n",
    "cancelled_train = pd.read_csv('../data/cancelled_train.csv', index_col=0).reset_index(drop=True)\n",
    "cancelled_train = cancelled_train[cancelled_train['case:concept:name'].isin(sample(list(cancelled_train['case:concept:name'].unique()), 3000))]\n",
    "\n",
    "cancelled_test = pd.read_csv('../data/cancelled_test.csv', index_col=0).reset_index(drop=True)\n",
    "cancelled_test = cancelled_test[cancelled_test['case:concept:name'].isin(sample(list(cancelled_test['case:concept:name'].unique()), 700))]\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# https://pandas.pydata.org/pandas-docs/stable/user_guide/reshaping.html#reshaping-dummies to create the the categorical encoding\n",
    "# https://stackoverflow.com/questions/49161120/pandas-python-set-value-of-one-column-based-on-value-in-another-column to assign values of the result DF\n",
    "# https://stackoverflow.com/questions/71426679/cumulative-sum-of-time-from-timestamps-in-pandas for cumulative time for boris"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Define preprocessing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "train = pd.concat([approved_train, declined_train, cancelled_train])\n",
    "train['event_w_lifecycle'] = train['concept:name'] + ' ' + train['lifecycle:transition']\n",
    "\n",
    "\n",
    "org_resource_cols = list(train['org:resource'].unique())\n",
    "event_cols = list(train['event_w_lifecycle'].unique())\n",
    "rest_cols = ['FirstWithdrawalAmount',\n",
    "       'NumberOfTerms', 'Accepted', 'MonthlyCost', 'Selected', 'CreditScore',\n",
    "       'OfferedAmount', 'case:LoanGoal', 'case:ApplicationType', 'case:RequestedAmount']\n",
    "target_cols = ['first_timestamp', 'trace_duration', 'case_outcome', 'case_progression']\n",
    "\n",
    "res_cols = org_resource_cols + event_cols + rest_cols + target_cols\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "def aggregate_df(df, res_cols):\n",
    "    \"\"\" \n",
    "    Aggregate the df of current events in the case\n",
    "\n",
    "    Output: \n",
    "        result -> could be a pandas series\n",
    "    \"\"\"\n",
    "    res_dict = dict.fromkeys(res_cols)\n",
    "    \n",
    "    for row in df.to_dict('records'):\n",
    "\n",
    "        # user variable assign\n",
    "        user = row['org:resource']\n",
    "        # event variable assign\n",
    "        event = row['concept:name'] + ' ' + row['lifecycle:transition']\n",
    "\n",
    "        try:\n",
    "            # first mention, turn it numeric\n",
    "            if not res_dict[user]: res_dict[user] = 1\n",
    "            else: res_dict[user] += 1\n",
    "\n",
    "            if not res_dict[event]: res_dict[event] = 1\n",
    "            else: res_dict[event] += 1\n",
    "        except:\n",
    "            # when test set has a unique resource (user) or event not in train set\n",
    "            pass\n",
    "\n",
    "        # case level cols\n",
    "        rest_cols_1 = ['FirstWithdrawalAmount',\n",
    "       'NumberOfTerms', 'Accepted', 'MonthlyCost', 'Selected', 'CreditScore',\n",
    "       'OfferedAmount'] \n",
    "\n",
    "        for col in rest_cols_1: # always get the newest record\n",
    "            if not math.isnan(row[col]) and row[col] != res_dict[col]:\n",
    "                res_dict[col] = row[col]\n",
    "\n",
    "\n",
    "        \n",
    "        # time stuff\n",
    "        #keeping the first timestamp of the case for calculation purposes\n",
    "        if not res_dict['first_timestamp']: res_dict['first_timestamp'] = row['time:timestamp']\n",
    "\n",
    "\n",
    "    rest_cols_2 = ['case:LoanGoal', 'case:ApplicationType', 'case:RequestedAmount']\n",
    "    for col in rest_cols_2:\n",
    "          res_dict[col] = row[col]  \n",
    "    \n",
    "    res_dict['case_progression'] = 0\n",
    "    \n",
    "    # trace duration in seconds\n",
    "    res_dict['trace_duration'] = (np.datetime64(row['time:timestamp']) - np.datetime64(res_dict['first_timestamp'])).item().total_seconds()\n",
    "\n",
    "    return res_dict\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "def add_to_aggregate(result, df_row):\n",
    "    \"\"\" \n",
    "    When a new event happens, add the event info to the current aggregated result.\n",
    "\n",
    "    Input: \n",
    "         result: the current aggregated result\n",
    "         df_row: pandas df row representing the new event\n",
    "    Output:\n",
    "        result: the new aggregated result\n",
    "    \"\"\"\n",
    "    last_row = result[-1].copy()\n",
    "    new_event = df_row.to_dict()\n",
    "    new_row = last_row\n",
    "\n",
    "    user = new_event['org:resource']\n",
    "    # event variable assign\n",
    "    event = new_event['concept:name'] + ' ' + new_event['lifecycle:transition']\n",
    "\n",
    "    try:\n",
    "        # first mention, turn it numeric\n",
    "        if not new_row[user]: new_row[user] = 1\n",
    "        else: new_row[user] += 1\n",
    "\n",
    "        if not new_row[event]: new_row[event] = 1\n",
    "        else: new_row[event] += 1\n",
    "    except:\n",
    "        # when test set has a unique resource (user) or event not in train set\n",
    "        pass\n",
    "\n",
    "    # case level cols\n",
    "    rest_cols_1 = ['FirstWithdrawalAmount',\n",
    "    'NumberOfTerms', 'Accepted', 'MonthlyCost', 'Selected', 'CreditScore',\n",
    "    'OfferedAmount'] \n",
    "\n",
    "    for col in rest_cols_1: # always get the newest record\n",
    "        if not math.isnan(new_event[col]) and new_event[col] != new_row[col]:\n",
    "            new_row[col] = new_event[col]\n",
    "    \n",
    "    rest_cols_2 = ['case:LoanGoal', 'case:ApplicationType', 'case:RequestedAmount']\n",
    "    for col in rest_cols_2:\n",
    "          new_row[col] = new_event[col]  \n",
    "\n",
    "\n",
    "    # time stuff\n",
    "    # trace duration in seconds\n",
    "    new_row['trace_duration'] = (np.datetime64(new_event['time:timestamp']) - np.datetime64(new_row['first_timestamp'])).item().total_seconds()\n",
    "\n",
    "    return new_row"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_prefix_part2(cases_df, res_cols, end_event, start_event='A_Accepted'):\n",
    "    \n",
    "    app_id_list = list(cases_df['case:concept:name'].unique())\n",
    "\n",
    "    # TODO:\n",
    "    # create a return df\n",
    "    return_list = []\n",
    "\n",
    "    # extracting prefix for each application\n",
    "    for app_id in app_id_list:\n",
    "        \n",
    "        events_app = cases_df.loc[cases_df['case:concept:name'] == app_id]\n",
    "        events_app.reset_index(drop=True, inplace=True)\n",
    "\n",
    "        cur_id = starting_row_id = events_app.loc[events_app['concept:name'] == 'A_Accepted'].index[0]\n",
    "        pre_events = events_app.iloc[:starting_row_id]\n",
    "        # TODO: \n",
    "        # aggregate events_app from row 0 to starting_row_id\n",
    "        return_list.append(aggregate_df(pre_events, res_cols=res_cols))\n",
    "        \n",
    "        ending_row_id = events_app.loc[events_app['concept:name'] == end_event].index[0] -1\n",
    "        cur_id += 1\n",
    "        \n",
    "        total_events = ending_row_id - starting_row_id - 1\n",
    "\n",
    "\n",
    "        while cur_id < ending_row_id:\n",
    "            new_row = events_app.iloc[cur_id]\n",
    "            # TODO: \n",
    "            # add new event row info to the aggregated result\n",
    "            d = add_to_aggregate(return_list, new_row)\n",
    "            d['case_progression'] = float(cur_id - starting_row_id)/total_events\n",
    "            return_list.append(d)\n",
    "\n",
    "            # Update the return_df -> add new row\n",
    "            # target y: end_event\n",
    "\n",
    "            cur_id += 1\n",
    "\n",
    "    result = pd.DataFrame.from_dict(return_list)\n",
    "    result['case_outcome'] = end_event\n",
    "    return result"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Run preprocessing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/var/folders/9l/kwd1671542z9tyvsgbp6mmtr0000gn/T/ipykernel_22252/260537720.py:51: DeprecationWarning: parsing timezone aware datetimes is deprecated; this will raise an error in the future\n",
      "  res_dict['trace_duration'] = (np.datetime64(row['time:timestamp']) - np.datetime64(res_dict['first_timestamp'])).item().total_seconds()\n",
      "/var/folders/9l/kwd1671542z9tyvsgbp6mmtr0000gn/T/ipykernel_22252/842079570.py:46: DeprecationWarning: parsing timezone aware datetimes is deprecated; this will raise an error in the future\n",
      "  new_row['trace_duration'] = (np.datetime64(new_event['time:timestamp']) - np.datetime64(new_row['first_timestamp'])).item().total_seconds()\n"
     ]
    }
   ],
   "source": [
    "prefix_cancelled_train = create_prefix_part2(cancelled_train, res_cols, end_event='A_Cancelled')\n",
    "prefix_cancelled_train.to_csv('../data/prefix_data/prefix_cancelled_train.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/var/folders/9l/kwd1671542z9tyvsgbp6mmtr0000gn/T/ipykernel_22252/260537720.py:51: DeprecationWarning: parsing timezone aware datetimes is deprecated; this will raise an error in the future\n",
      "  res_dict['trace_duration'] = (np.datetime64(row['time:timestamp']) - np.datetime64(res_dict['first_timestamp'])).item().total_seconds()\n",
      "/var/folders/9l/kwd1671542z9tyvsgbp6mmtr0000gn/T/ipykernel_22252/842079570.py:46: DeprecationWarning: parsing timezone aware datetimes is deprecated; this will raise an error in the future\n",
      "  new_row['trace_duration'] = (np.datetime64(new_event['time:timestamp']) - np.datetime64(new_row['first_timestamp'])).item().total_seconds()\n"
     ]
    }
   ],
   "source": [
    "prefix_cancelled_test = create_prefix_part2(cancelled_test, res_cols, end_event='A_Cancelled')\n",
    "prefix_cancelled_train.to_csv('../data/prefix_data/prefix_cancelled_test.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/var/folders/9l/kwd1671542z9tyvsgbp6mmtr0000gn/T/ipykernel_22252/260537720.py:51: DeprecationWarning: parsing timezone aware datetimes is deprecated; this will raise an error in the future\n",
      "  res_dict['trace_duration'] = (np.datetime64(row['time:timestamp']) - np.datetime64(res_dict['first_timestamp'])).item().total_seconds()\n",
      "/var/folders/9l/kwd1671542z9tyvsgbp6mmtr0000gn/T/ipykernel_22252/842079570.py:46: DeprecationWarning: parsing timezone aware datetimes is deprecated; this will raise an error in the future\n",
      "  new_row['trace_duration'] = (np.datetime64(new_event['time:timestamp']) - np.datetime64(new_row['first_timestamp'])).item().total_seconds()\n",
      "/var/folders/9l/kwd1671542z9tyvsgbp6mmtr0000gn/T/ipykernel_22252/260537720.py:51: DeprecationWarning: parsing timezone aware datetimes is deprecated; this will raise an error in the future\n",
      "  res_dict['trace_duration'] = (np.datetime64(row['time:timestamp']) - np.datetime64(res_dict['first_timestamp'])).item().total_seconds()\n",
      "/var/folders/9l/kwd1671542z9tyvsgbp6mmtr0000gn/T/ipykernel_22252/842079570.py:46: DeprecationWarning: parsing timezone aware datetimes is deprecated; this will raise an error in the future\n",
      "  new_row['trace_duration'] = (np.datetime64(new_event['time:timestamp']) - np.datetime64(new_row['first_timestamp'])).item().total_seconds()\n"
     ]
    }
   ],
   "source": [
    "prefix_approved_train = create_prefix_part2(approved_train, res_cols, \"A_Pending\")\n",
    "prefix_approved_train.to_csv('../data/prefix_data/prefix_approved_train.csv')\n",
    "\n",
    "prefix_approved_test = create_prefix_part2(approved_test, res_cols, \"A_Pending\")\n",
    "prefix_approved_test.to_csv('../data/prefix_data/prefix_approved_test.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/var/folders/9l/kwd1671542z9tyvsgbp6mmtr0000gn/T/ipykernel_22252/260537720.py:51: DeprecationWarning: parsing timezone aware datetimes is deprecated; this will raise an error in the future\n",
      "  res_dict['trace_duration'] = (np.datetime64(row['time:timestamp']) - np.datetime64(res_dict['first_timestamp'])).item().total_seconds()\n",
      "/var/folders/9l/kwd1671542z9tyvsgbp6mmtr0000gn/T/ipykernel_22252/842079570.py:46: DeprecationWarning: parsing timezone aware datetimes is deprecated; this will raise an error in the future\n",
      "  new_row['trace_duration'] = (np.datetime64(new_event['time:timestamp']) - np.datetime64(new_row['first_timestamp'])).item().total_seconds()\n",
      "/var/folders/9l/kwd1671542z9tyvsgbp6mmtr0000gn/T/ipykernel_22252/260537720.py:51: DeprecationWarning: parsing timezone aware datetimes is deprecated; this will raise an error in the future\n",
      "  res_dict['trace_duration'] = (np.datetime64(row['time:timestamp']) - np.datetime64(res_dict['first_timestamp'])).item().total_seconds()\n",
      "/var/folders/9l/kwd1671542z9tyvsgbp6mmtr0000gn/T/ipykernel_22252/842079570.py:46: DeprecationWarning: parsing timezone aware datetimes is deprecated; this will raise an error in the future\n",
      "  new_row['trace_duration'] = (np.datetime64(new_event['time:timestamp']) - np.datetime64(new_row['first_timestamp'])).item().total_seconds()\n"
     ]
    }
   ],
   "source": [
    "prefix_declined_train = create_prefix_part2(declined_train, res_cols, 'A_Denied')\n",
    "prefix_declined_train.to_csv('../data/prefix_data/prefix_declined_train.csv')\n",
    "\n",
    "prefix_declined_test = create_prefix_part2(declined_test, res_cols, 'A_Denied')\n",
    "prefix_declined_test.to_csv('../data/prefix_data/prefix_declined_test.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "prefix_train = pd.concat([prefix_approved_train, prefix_cancelled_train, prefix_declined_train])\n",
    "prefix_train.to_csv('../data/prefix_data/full_prefix_train.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "prefix_test = pd.concat([prefix_approved_test, prefix_cancelled_test, prefix_declined_test])\n",
    "prefix_test.to_csv('../data/prefix_data/full_prefix_test.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.10.8 ('RT-PM-T9gc75cv')",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.8"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "c6704c5f6fbd5bde55bd38fea2be55cd8f7e1d4452f228187dcec36853dd5375"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}