Something went wrong. Try again.
[READ-ONLY] Mirror of https://github.com/agbocsardi/rtpm_assignment_2.
Something went wrong. Try again.
18 kB · 436 lines
at main
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437{ "cells": [ { "cell_type": "code", "execution_count": 28, "metadata": {}, "outputs": [], "source": [ "import pandas as pd\n", "# import pm4py\n", "from tqdm import tqdm\n", "import os\n", "import numpy as np\n", "import math" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Importing data" ] }, { "cell_type": "code", "execution_count": 31, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "712\n", "3001\n" ] } ], "source": [ "# declined_train = pd.read_csv('../data/declined_train.csv')\n", "declined_test = pd.read_csv('../data/declined_test.csv', index_col=0).reset_index(drop=True)\n", "print(len(declined_test['case:concept:name'].unique()))\n", "\n", "declined_train = pd.read_csv('../data/declined_train.csv', index_col=0).reset_index(drop=True)\n", "print(len(declined_train['case:concept:name'].unique()))" ] }, { "cell_type": "code", "execution_count": 32, "metadata": {}, "outputs": [], "source": [ "from random import sample\n", "approved_train = pd.read_csv('../data/approved_train.csv', index_col=0).reset_index(drop=True)\n", "approved_train = approved_train[approved_train['case:concept:name'].isin(sample(list(approved_train['case:concept:name'].unique()), 3000))]\n", "\n", "approved_test = pd.read_csv('../data/approved_test.csv', index_col=0).reset_index(drop=True)\n", "approved_test = approved_test[approved_test['case:concept:name'].isin(sample(list(approved_test['case:concept:name'].unique()), 700))]\n", "\n", "\n", "\n", "cancelled_train = pd.read_csv('../data/cancelled_train.csv', index_col=0).reset_index(drop=True)\n", "cancelled_train = cancelled_train[cancelled_train['case:concept:name'].isin(sample(list(cancelled_train['case:concept:name'].unique()), 3000))]\n", "\n", "cancelled_test = pd.read_csv('../data/cancelled_test.csv', index_col=0).reset_index(drop=True)\n", "cancelled_test = cancelled_test[cancelled_test['case:concept:name'].isin(sample(list(cancelled_test['case:concept:name'].unique()), 700))]\n", "\n" ] }, { "cell_type": "code", "execution_count": 6, "metadata": {}, "outputs": [], "source": [ "# https://pandas.pydata.org/pandas-docs/stable/user_guide/reshaping.html#reshaping-dummies to create the the categorical encoding\n", "# https://stackoverflow.com/questions/49161120/pandas-python-set-value-of-one-column-based-on-value-in-another-column to assign values of the result DF\n", "# https://stackoverflow.com/questions/71426679/cumulative-sum-of-time-from-timestamps-in-pandas for cumulative time for boris" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Define preprocessing" ] }, { "cell_type": "code", "execution_count": 35, "metadata": {}, "outputs": [], "source": [ "train = pd.concat([approved_train, declined_train, cancelled_train])\n", "train['event_w_lifecycle'] = train['concept:name'] + ' ' + train['lifecycle:transition']\n", "\n", "\n", "org_resource_cols = list(train['org:resource'].unique())\n", "event_cols = list(train['event_w_lifecycle'].unique())\n", "rest_cols = ['FirstWithdrawalAmount',\n", " 'NumberOfTerms', 'Accepted', 'MonthlyCost', 'Selected', 'CreditScore',\n", " 'OfferedAmount', 'case:LoanGoal', 'case:ApplicationType', 'case:RequestedAmount']\n", "target_cols = ['first_timestamp', 'trace_duration', 'case_outcome', 'case_progression']\n", "\n", "res_cols = org_resource_cols + event_cols + rest_cols + target_cols\n" ] }, { "cell_type": "code", "execution_count": 40, "metadata": {}, "outputs": [], "source": [ "def aggregate_df(df, res_cols):\n", " \"\"\" \n", " Aggregate the df of current events in the case\n", "\n", " Output: \n", " result -> could be a pandas series\n", " \"\"\"\n", " res_dict = dict.fromkeys(res_cols)\n", " \n", " for row in df.to_dict('records'):\n", "\n", " # user variable assign\n", " user = row['org:resource']\n", " # event variable assign\n", " event = row['concept:name'] + ' ' + row['lifecycle:transition']\n", "\n", " try:\n", " # first mention, turn it numeric\n", " if not res_dict[user]: res_dict[user] = 1\n", " else: res_dict[user] += 1\n", "\n", " if not res_dict[event]: res_dict[event] = 1\n", " else: res_dict[event] += 1\n", " except:\n", " # when test set has a unique resource (user) or event not in train set\n", " pass\n", "\n", " # case level cols\n", " rest_cols_1 = ['FirstWithdrawalAmount',\n", " 'NumberOfTerms', 'Accepted', 'MonthlyCost', 'Selected', 'CreditScore',\n", " 'OfferedAmount'] \n", "\n", " for col in rest_cols_1: # always get the newest record\n", " if not math.isnan(row[col]) and row[col] != res_dict[col]:\n", " res_dict[col] = row[col]\n", "\n", "\n", " \n", " # time stuff\n", " #keeping the first timestamp of the case for calculation purposes\n", " if not res_dict['first_timestamp']: res_dict['first_timestamp'] = row['time:timestamp']\n", "\n", "\n", " rest_cols_2 = ['case:LoanGoal', 'case:ApplicationType', 'case:RequestedAmount']\n", " for col in rest_cols_2:\n", " res_dict[col] = row[col] \n", " \n", " res_dict['case_progression'] = 0\n", " \n", " # trace duration in seconds\n", " res_dict['trace_duration'] = (np.datetime64(row['time:timestamp']) - np.datetime64(res_dict['first_timestamp'])).item().total_seconds()\n", "\n", " return res_dict\n", "\n" ] }, { "cell_type": "code", "execution_count": 38, "metadata": {}, "outputs": [], "source": [ "def add_to_aggregate(result, df_row):\n", " \"\"\" \n", " When a new event happens, add the event info to the current aggregated result.\n", "\n", " Input: \n", " result: the current aggregated result\n", " df_row: pandas df row representing the new event\n", " Output:\n", " result: the new aggregated result\n", " \"\"\"\n", " last_row = result[-1].copy()\n", " new_event = df_row.to_dict()\n", " new_row = last_row\n", "\n", " user = new_event['org:resource']\n", " # event variable assign\n", " event = new_event['concept:name'] + ' ' + new_event['lifecycle:transition']\n", "\n", " try:\n", " # first mention, turn it numeric\n", " if not new_row[user]: new_row[user] = 1\n", " else: new_row[user] += 1\n", "\n", " if not new_row[event]: new_row[event] = 1\n", " else: new_row[event] += 1\n", " except:\n", " # when test set has a unique resource (user) or event not in train set\n", " pass\n", "\n", " # case level cols\n", " rest_cols_1 = ['FirstWithdrawalAmount',\n", " 'NumberOfTerms', 'Accepted', 'MonthlyCost', 'Selected', 'CreditScore',\n", " 'OfferedAmount'] \n", "\n", " for col in rest_cols_1: # always get the newest record\n", " if not math.isnan(new_event[col]) and new_event[col] != new_row[col]:\n", " new_row[col] = new_event[col]\n", " \n", " rest_cols_2 = ['case:LoanGoal', 'case:ApplicationType', 'case:RequestedAmount']\n", " for col in rest_cols_2:\n", " new_row[col] = new_event[col] \n", "\n", "\n", " # time stuff\n", " # trace duration in seconds\n", " new_row['trace_duration'] = (np.datetime64(new_event['time:timestamp']) - np.datetime64(new_row['first_timestamp'])).item().total_seconds()\n", "\n", " return new_row" ] }, { "cell_type": "code", "execution_count": 39, "metadata": {}, "outputs": [], "source": [ "def create_prefix_part2(cases_df, res_cols, end_event, start_event='A_Accepted'):\n", " \n", " app_id_list = list(cases_df['case:concept:name'].unique())\n", "\n", " # TODO:\n", " # create a return df\n", " return_list = []\n", "\n", " # extracting prefix for each application\n", " for app_id in app_id_list:\n", " \n", " events_app = cases_df.loc[cases_df['case:concept:name'] == app_id]\n", " events_app.reset_index(drop=True, inplace=True)\n", "\n", " cur_id = starting_row_id = events_app.loc[events_app['concept:name'] == 'A_Accepted'].index[0]\n", " pre_events = events_app.iloc[:starting_row_id]\n", " # TODO: \n", " # aggregate events_app from row 0 to starting_row_id\n", " return_list.append(aggregate_df(pre_events, res_cols=res_cols))\n", " \n", " ending_row_id = events_app.loc[events_app['concept:name'] == end_event].index[0] -1\n", " cur_id += 1\n", " \n", " total_events = ending_row_id - starting_row_id - 1\n", "\n", "\n", " while cur_id < ending_row_id:\n", " new_row = events_app.iloc[cur_id]\n", " # TODO: \n", " # add new event row info to the aggregated result\n", " d = add_to_aggregate(return_list, new_row)\n", " d['case_progression'] = float(cur_id - starting_row_id)/total_events\n", " return_list.append(d)\n", "\n", " # Update the return_df -> add new row\n", " # target y: end_event\n", "\n", " cur_id += 1\n", "\n", " result = pd.DataFrame.from_dict(return_list)\n", " result['case_outcome'] = end_event\n", " return result" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Run preprocessing" ] }, { "cell_type": "code", "execution_count": 45, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "/var/folders/9l/kwd1671542z9tyvsgbp6mmtr0000gn/T/ipykernel_22252/260537720.py:51: DeprecationWarning: parsing timezone aware datetimes is deprecated; this will raise an error in the future\n", " res_dict['trace_duration'] = (np.datetime64(row['time:timestamp']) - np.datetime64(res_dict['first_timestamp'])).item().total_seconds()\n", "/var/folders/9l/kwd1671542z9tyvsgbp6mmtr0000gn/T/ipykernel_22252/842079570.py:46: DeprecationWarning: parsing timezone aware datetimes is deprecated; this will raise an error in the future\n", " new_row['trace_duration'] = (np.datetime64(new_event['time:timestamp']) - np.datetime64(new_row['first_timestamp'])).item().total_seconds()\n" ] } ], "source": [ "prefix_cancelled_train = create_prefix_part2(cancelled_train, res_cols, end_event='A_Cancelled')\n", "prefix_cancelled_train.to_csv('../data/prefix_data/prefix_cancelled_train.csv')" ] }, { "cell_type": "code", "execution_count": 46, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "/var/folders/9l/kwd1671542z9tyvsgbp6mmtr0000gn/T/ipykernel_22252/260537720.py:51: DeprecationWarning: parsing timezone aware datetimes is deprecated; this will raise an error in the future\n", " res_dict['trace_duration'] = (np.datetime64(row['time:timestamp']) - np.datetime64(res_dict['first_timestamp'])).item().total_seconds()\n", "/var/folders/9l/kwd1671542z9tyvsgbp6mmtr0000gn/T/ipykernel_22252/842079570.py:46: DeprecationWarning: parsing timezone aware datetimes is deprecated; this will raise an error in the future\n", " new_row['trace_duration'] = (np.datetime64(new_event['time:timestamp']) - np.datetime64(new_row['first_timestamp'])).item().total_seconds()\n" ] } ], "source": [ "prefix_cancelled_test = create_prefix_part2(cancelled_test, res_cols, end_event='A_Cancelled')\n", "prefix_cancelled_train.to_csv('../data/prefix_data/prefix_cancelled_test.csv')" ] }, { "cell_type": "code", "execution_count": 47, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "/var/folders/9l/kwd1671542z9tyvsgbp6mmtr0000gn/T/ipykernel_22252/260537720.py:51: DeprecationWarning: parsing timezone aware datetimes is deprecated; this will raise an error in the future\n", " res_dict['trace_duration'] = (np.datetime64(row['time:timestamp']) - np.datetime64(res_dict['first_timestamp'])).item().total_seconds()\n", "/var/folders/9l/kwd1671542z9tyvsgbp6mmtr0000gn/T/ipykernel_22252/842079570.py:46: DeprecationWarning: parsing timezone aware datetimes is deprecated; this will raise an error in the future\n", " new_row['trace_duration'] = (np.datetime64(new_event['time:timestamp']) - np.datetime64(new_row['first_timestamp'])).item().total_seconds()\n", "/var/folders/9l/kwd1671542z9tyvsgbp6mmtr0000gn/T/ipykernel_22252/260537720.py:51: DeprecationWarning: parsing timezone aware datetimes is deprecated; this will raise an error in the future\n", " res_dict['trace_duration'] = (np.datetime64(row['time:timestamp']) - np.datetime64(res_dict['first_timestamp'])).item().total_seconds()\n", "/var/folders/9l/kwd1671542z9tyvsgbp6mmtr0000gn/T/ipykernel_22252/842079570.py:46: DeprecationWarning: parsing timezone aware datetimes is deprecated; this will raise an error in the future\n", " new_row['trace_duration'] = (np.datetime64(new_event['time:timestamp']) - np.datetime64(new_row['first_timestamp'])).item().total_seconds()\n" ] } ], "source": [ "prefix_approved_train = create_prefix_part2(approved_train, res_cols, \"A_Pending\")\n", "prefix_approved_train.to_csv('../data/prefix_data/prefix_approved_train.csv')\n", "\n", "prefix_approved_test = create_prefix_part2(approved_test, res_cols, \"A_Pending\")\n", "prefix_approved_test.to_csv('../data/prefix_data/prefix_approved_test.csv')" ] }, { "cell_type": "code", "execution_count": 49, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "/var/folders/9l/kwd1671542z9tyvsgbp6mmtr0000gn/T/ipykernel_22252/260537720.py:51: DeprecationWarning: parsing timezone aware datetimes is deprecated; this will raise an error in the future\n", " res_dict['trace_duration'] = (np.datetime64(row['time:timestamp']) - np.datetime64(res_dict['first_timestamp'])).item().total_seconds()\n", "/var/folders/9l/kwd1671542z9tyvsgbp6mmtr0000gn/T/ipykernel_22252/842079570.py:46: DeprecationWarning: parsing timezone aware datetimes is deprecated; this will raise an error in the future\n", " new_row['trace_duration'] = (np.datetime64(new_event['time:timestamp']) - np.datetime64(new_row['first_timestamp'])).item().total_seconds()\n", "/var/folders/9l/kwd1671542z9tyvsgbp6mmtr0000gn/T/ipykernel_22252/260537720.py:51: DeprecationWarning: parsing timezone aware datetimes is deprecated; this will raise an error in the future\n", " res_dict['trace_duration'] = (np.datetime64(row['time:timestamp']) - np.datetime64(res_dict['first_timestamp'])).item().total_seconds()\n", "/var/folders/9l/kwd1671542z9tyvsgbp6mmtr0000gn/T/ipykernel_22252/842079570.py:46: DeprecationWarning: parsing timezone aware datetimes is deprecated; this will raise an error in the future\n", " new_row['trace_duration'] = (np.datetime64(new_event['time:timestamp']) - np.datetime64(new_row['first_timestamp'])).item().total_seconds()\n" ] } ], "source": [ "prefix_declined_train = create_prefix_part2(declined_train, res_cols, 'A_Denied')\n", "prefix_declined_train.to_csv('../data/prefix_data/prefix_declined_train.csv')\n", "\n", "prefix_declined_test = create_prefix_part2(declined_test, res_cols, 'A_Denied')\n", "prefix_declined_test.to_csv('../data/prefix_data/prefix_declined_test.csv')" ] }, { "cell_type": "code", "execution_count": 51, "metadata": {}, "outputs": [], "source": [ "prefix_train = pd.concat([prefix_approved_train, prefix_cancelled_train, prefix_declined_train])\n", "prefix_train.to_csv('../data/prefix_data/full_prefix_train.csv')" ] }, { "cell_type": "code", "execution_count": 53, "metadata": {}, "outputs": [], "source": [ "prefix_test = pd.concat([prefix_approved_test, prefix_cancelled_test, prefix_declined_test])\n", "prefix_test.to_csv('../data/prefix_data/full_prefix_test.csv')" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3.10.8 ('RT-PM-T9gc75cv')", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.10.8" }, "orig_nbformat": 4, "vscode": { "interpreter": { "hash": "c6704c5f6fbd5bde55bd38fea2be55cd8f7e1d4452f228187dcec36853dd5375" } } }, "nbformat": 4, "nbformat_minor": 2}