kleister-nda-clone/kleister-nda.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Extract key information from Edgar NDA documents"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pathlib\n",
    "from collections import Counter\n",
    "from sklearn.metrics import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "KLEISTER_PATH = pathlib.Path('C:/Users/Fijka/Documents/kleister-nda-clone')\n",
    "file_name = 'train'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Read expected train data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_expected_data(filepath, data_key):\n",
    "    dataset_expected_key = []\n",
    "    with open(filepath,'r') as train_expected_file:\n",
    "        for line in train_expected_file:\n",
    "            key_values = line.rstrip('\\n').split(' ')\n",
    "            data_value = None\n",
    "            for key_value in key_values:\n",
    "                key, value = key_value.split('=')\n",
    "                if key == data_key:\n",
    "                    data_value = value\n",
    "            if data_value is None:\n",
    "                data_value = 'NONE'\n",
    "            dataset_expected_key.append(data_value)\n",
    "    return dataset_expected_key"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "KEYS = ['effective_date', 'jurisdiction', 'party', 'term']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def read_expected_data(filepath):\n",
    "    data = []\n",
    "    for key in KEYS:\n",
    "        data.append(get_expected_data(filepath, key))\n",
    "    return data\n",
    "\n",
    "if file_name != 'test-A':\n",
    "    train_expected_data = read_expected_data(KLEISTER_PATH/file_name/'expected.tsv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "if file_name != 'test-A':\n",
    "    [i[:1] for i in train_expected_data]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Read train dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "import lzma\n",
    "import csv\n",
    "\n",
    "def read_data(filename):\n",
    "    all_data = lzma.open(filename).read().decode('UTF-8').split('\\n')\n",
    "    return [line.split('\\t') for line in all_data][:-1]\n",
    "\n",
    "train_data = read_data(KLEISTER_PATH/file_name/'in.tsv.xz')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## JURISDICTION"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "STATES = ['Alabama', 'Alaska', 'Arizona', 'Arkansas', 'California', 'Colorado', 'Connecticut', 'Delaware','Florida',\n",
    "          'Georgia', 'Hawaii', 'Idaho', 'Illinois', 'Indiana', 'Iowa', 'Kansas', 'Kentucky', 'Louisiana', 'Maine',\n",
    "          'Maryland', 'Massachusetts', 'Michigan', 'Minnesota', 'Mississippi', 'Missouri', 'Montana', 'Nebraska', 'Nevada',\n",
    "          'New Hampshire', 'New Jersey', 'New Mexico', 'New York', 'North Carolina', 'North Dakota', 'Ohio', 'Oklahoma',\n",
    "          'Oregon', 'Pennsylvania', 'Rhode Island', 'South Carolina', 'South Dakota', 'Tennessee', 'Texas', 'Utah',\n",
    "          'Vermont', 'Virginia', 'Washington', 'West Virginia', 'Wisconsin', 'Wyoming']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "false jurisdiction: 22\n"
     ]
    }
   ],
   "source": [
    "import spacy\n",
    "nlp = spacy.load(\"en_core_web_sm\")\n",
    "from operator import itemgetter\n",
    "\n",
    "jurisdiction = []\n",
    "\n",
    "def normalize(text):\n",
    "    return text.replace('\\\\n', ' ').lower()\n",
    "    # nlp(text) -> tokenizacja\n",
    "\n",
    "def check_jurisdiction(document):\n",
    "    states = {}\n",
    "    for text in document[2:]:\n",
    "        text = normalize(text)\n",
    "        for state in STATES:\n",
    "            if state.lower() in text:\n",
    "                if state in states:\n",
    "                    states[state][0] += text.count(state.lower())\n",
    "                else:\n",
    "                    states[state] = [text.count(state.lower()), text.index(state.lower())]\n",
    "    if states != {}:\n",
    "        states = sorted(states.items(), key=itemgetter(1), reverse=True)\n",
    "        jurisdiction.append(states[0][0].replace(' ', '_'))\n",
    "        return states[0][0], states\n",
    "    else:\n",
    "        jurisdiction.append(None)\n",
    "        return None\n",
    "    \n",
    "tmp = 0\n",
    "for i in range(len(train_data)):\n",
    "    tt = check_jurisdiction(train_data[i])\n",
    "    if file_name != 'test-A':\n",
    "        if tt == None:\n",
    "            if train_expected_data[1][i] != None:\n",
    "    #             print(i, train_expected_data[1][i], tt)\n",
    "                tmp += 1\n",
    "        else:\n",
    "            if tt[0] != train_expected_data[1][i].replace('_', ' '):\n",
    "    #             print(i, train_expected_data[1][i], tt[0])\n",
    "                tmp += 1\n",
    "print('false jurisdiction:', tmp)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## EFFECTIVE DATE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "false effective date 42\n"
     ]
    }
   ],
   "source": [
    "import re\n",
    "import datetime\n",
    "from datetime import date\n",
    "\n",
    "effective_date = []\n",
    "\n",
    "def parse_date(date):\n",
    "    month = str(date.month)\n",
    "    if len(month) == 1:\n",
    "        month = '0' + str(date.month)\n",
    "    day = str(date.day)\n",
    "    if len(day) == 1:\n",
    "        day = '0' + str(date.day)\n",
    "    return str(date.year) + '-' + month + '-' + day\n",
    "\n",
    "def find_dates(text):\n",
    "    \n",
    "    MONTHS = {'January' : 1, 'February' : 2, 'March' : 3, 'April' : 4, 'May' : 5, 'June' : 6,\n",
    "              'July' : 7, 'August' : 8, 'September' : 9, 'October' : 10, 'November' : 11, 'December' : 12}\n",
    "    \n",
    "    all_dates = []\n",
    "    \n",
    "    text = text.replace('\\\\n', ' ')\n",
    "    \n",
    "    dic = {'\\d{1,2}\\/\\d{1,2}\\/\\d{2}' : '%m/%d/%y',\n",
    "           '[01]*[0-9]\\/[01]*[0-9]\\/\\d{4}' : '%m/%d/%Y',\n",
    "           '\\w{3,9}?\\s\\d{1,2}?,\\s\\d{4}?' : '%B %d, %Y',\n",
    "           '\\w{3,9}?\\s\\d{1,2}?,\\d{4}?' : '%B %d,%Y',\n",
    "           '\\d{1,2}?th\\sday\\sof\\s\\w{3,9}?\\s\\d{4}?' : '%dth day of %B %Y',\n",
    "           '\\d{1,2}?th\\sday\\sof\\s\\w{3,9}?,\\s\\d{4}?' : '%dth day of %B, %Y',\n",
    "           '\\d{1,2}?ND\\sday\\sof\\s\\w{3,9}?\\s\\d{4}?' : '%dND day of %B %Y',\n",
    "           '\\w{3,9}?\\s\\d{1,2}?th\\s,\\s\\d{4}?' : '%B %dth , %Y',\n",
    "           '\\w{3,9}?\\s\\d{1,2}?th,\\s\\d{4}?' : '%B %dth, %Y',\n",
    "           '\\d{1,2}?\\sday\\sof\\s\\w{3,9}?,\\s\\d{4}?' : '%d day of %B, %Y',\n",
    "           '\\w{3,9}?\\.\\s\\d{1,2}?,\\s\\d{4}?' : '%b. %d, %Y',\n",
    "           '\\d{1,2}?\\s\\w{3,9}?,\\s\\d{4}?' : '%d %B, %Y',\n",
    "           '\\d{1,2}?st\\sday\\sof\\s\\w{3,9}?\\s,\\s\\d{4}?' : '%dst day of %B , %Y',\n",
    "           '\\d{1,2}?st\\sday\\sof\\s\\w{3,9}?,\\s\\d{4}?' : '%dst day of %B, %Y',\n",
    "           '\\d{1,2}?nd\\sday\\sof\\s\\w{3,9}?,\\s\\d{4}?' : '%dnd day of %B, %Y',\n",
    "           '\\d{1,2}\\.\\d{1,2}\\.\\d{2,4}' : '%m.%d.%y'\n",
    "          }\n",
    "    \n",
    "    for d in dic:\n",
    "        match = re.search(r'' + d, text)\n",
    "        if match != None:\n",
    "            try:\n",
    "                date = datetime.datetime.strptime(match.group(), dic[d]).date()\n",
    "                all_dates.append(parse_date(date))\n",
    "            except:\n",
    "                pass\n",
    "                \n",
    "    return all_dates\n",
    "\n",
    "def check_effective_date(text):\n",
    "    dates = []\n",
    "    x = find_dates(text)\n",
    "    if x != []:\n",
    "        dates.append(x)\n",
    "    return(dates)\n",
    "\n",
    "test = 0\n",
    "for i in range(len(train_data)):\n",
    "    xx = check_effective_date(train_data[i][2])\n",
    "    if file_name != 'test-A':\n",
    "        if train_expected_data[0][i] == 'NONE':\n",
    "            if xx != []:\n",
    "    #             print(i, train_expected_data[0][i], xx[-1][0])\n",
    "                test += 1\n",
    "        else:\n",
    "            if xx != []:\n",
    "                if xx[0][-1] != train_expected_data[0][i]:\n",
    "    #                 print(i, train_expected_data[0][i], xx[-1][0])\n",
    "                    test +=1\n",
    "            else:\n",
    "    #             print(i, train_expected_data[0][i], xx)\n",
    "                test += 1\n",
    "    if xx != []:\n",
    "        effective_date.append(xx[-1][0])\n",
    "    else:\n",
    "        effective_date.append(None)\n",
    "print('false effective date', test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## PARTY"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "false party: 202\n"
     ]
    }
   ],
   "source": [
    "party = []\n",
    "\n",
    "def check_party(document):\n",
    "    dic = {'And_' : 4,\n",
    "           'From_' : 5,\n",
    "           'For' : 4,\n",
    "           'Between' : 8,\n",
    "           'With' : 5,\n",
    "           'Ceo' : 4,\n",
    "           'To' : 3,\n",
    "          }\n",
    "    \n",
    "    for text in document[2:]:\n",
    "        text = text.replace('\\\\n', ' ')\n",
    "        \n",
    "        result = None\n",
    "        match = re.search(r'\\w*\\s\\w*\\s\\w*,\\sInc\\.', text)\n",
    "        if match == None:\n",
    "            match = re.search(r'\\w*\\s\\w*\\s\\w*,\\sINC\\.', text)\n",
    "        if match != None:\n",
    "            result = match.group().title()\n",
    "            result = result.replace(',', '').replace(' ', '_')\n",
    "            for d in dic:\n",
    "                if d in result:\n",
    "                    result = result[result.index(d) + dic[d]:]\n",
    "            if result.startswith('_'):\n",
    "                result = result[1:]\n",
    "    return result\n",
    "    \n",
    "tmp = 0\n",
    "for i in range(len(train_data)):\n",
    "    tt = check_party(train_data[i])\n",
    "    party.append(tt)\n",
    "    if file_name != 'test-A':\n",
    "        if train_expected_data[2][i] != tt:\n",
    "            tmp += 1\n",
    "    #         print(i, train_expected_data[2][i], tt)\n",
    "print('false party:', tmp)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## TERM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "false term: 144\n"
     ]
    }
   ],
   "source": [
    "term = []\n",
    "\n",
    "def check_term(document):\n",
    "    \n",
    "    result = None\n",
    "    for text in document[2:]:\n",
    "        text = text.replace('\\\\n', ' ')\n",
    "    \n",
    "        \n",
    "        match = re.search(r'\\(\\d*\\)\\syears', text)\n",
    "        if match == None:\n",
    "            match = re.search(r'\\(\\d*\\)\\smonths', text)\n",
    "        if match != None:\n",
    "            result = match.group().replace('(', '').replace(')', '').replace(' ', '_')\n",
    "            return result\n",
    "    return result\n",
    "    \n",
    "tmp = 0\n",
    "for i in range(len(train_data)):\n",
    "    tt = check_term(train_data[i])\n",
    "    term.append(tt)\n",
    "    if file_name != 'test-A':\n",
    "        if train_expected_data[3][i] != tt:\n",
    "            if train_expected_data[3][i] == 'NONE' and tt == None:\n",
    "                pass\n",
    "            else:\n",
    "    #             print(i, train_expected_data[3][i], tt)\n",
    "                tmp += 1\n",
    "print('false term:', tmp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "def write_output(effective_date, jurisdiction, party, term):\n",
    "    if os.path.exists(KLEISTER_PATH/file_name/'out.tsv'):\n",
    "        os.remove(KLEISTER_PATH/file_name/'out.tsv')\n",
    "    file = open(KLEISTER_PATH/file_name/'out.tsv', 'w')\n",
    "    for doc in range(len(effective_date)):\n",
    "        result = ''\n",
    "        if effective_date[doc] != None:\n",
    "            result += 'effective_date=' + effective_date[doc] + '\\t'\n",
    "        if jurisdiction[doc] != None:\n",
    "            result += 'jurisdiction=' + jurisdiction[doc] + '\\t'\n",
    "        if party[doc] != None:\n",
    "            result += 'party=' + party[doc] + '\\t'\n",
    "        if term[doc] != None:\n",
    "            result += 'term=' + term[doc] + '\\t'\n",
    "        if len(result) > 1:\n",
    "            result = result[:-1]\n",
    "        result += '\\n'\n",
    "        file.write(result)\n",
    "    file.close()\n",
    "    \n",
    "write_output(effective_date, jurisdiction, party, term)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}