import csv import sys import lzma import os import regex as re out_file_name = "out.tsv" out_sep = '\t' months = { "January": "01", "February": "02", "March": "03", "April": "04", "May": "05", "June": "06", "July": "07", "August": "08", "September": "09", "October": "10", "November": "11", "December": "12" } def main(fname: str): dir_path = os.path.dirname(fname) out_path = os.path.join(dir_path, out_file_name) with lzma.open(fname, mode="rt", encoding="utf-8") as tsv_file: with open(out_path, "w") as out_file: reader = csv.reader(tsv_file, delimiter='\t') for item in reader: out_file.write(process_entry(item) + '\n') def process_entry(entry: list) -> str: res = "" parameters = entry[1] for p in parameters.split(" "): t = entry[2] if p == "party": party = get_party(t) if party is not None: res += out_sep + "party=" + party elif p == "jurisdiction": jur = get_jurisdiction(t) if jur is not None: res += out_sep + "jurisdiction=" + jur elif p == "effective_date": ed = get_date(t) if ed is not None: t_ed = format_date(ed) if t_ed is not None: res += out_sep + "effective_date=" + t_ed return res def get_party(text: str): res = None m = re.search( r"between([\p{L}\s.,]+)(inc.|Inc.|INC.|Llc|LLC|llc)[\,\.]", text) if m is not None: res = m.group(1).strip().replace(",", "").replace(" ", "_") t = m.group(2).lower() res += ("_" + t.replace("inc.", "Inc.").replace("llc", "LLC")) return res def get_jurisdiction(text: str): res = None m = re.search( r"laws? of the (?:(?:State of)|(?:Commonwealth of)) ([A-Z][a-z]+(?: [A-Z][a-z]+)?).+(?:(?: and)|[,.])", text) if m is not None: res = m.group(1).strip().replace(" ", "_") return res def get_date(text: str): res = re.search( r"(1[0-2]|0[1-9])[/-](0[1-9]|[1|2][0-9]|3[0-1])[/-](0[1-9]|[1-9][0-9])", text) if res is not None: return res res = re.search( r"(0[1-9]|1[1-2])[/-](0[1-9]|1[1-9]|2[1-9]|3[0-1])[/-](19[0-9][0-9]|20[0-9][0-9])", text) if res is not None: return res res = re.search( r"([j|J]anuary|[f|F]ebruary|[m|M]arch|[A|a]pril|[M|m]ay|[J|j]une|[J|j]uly|[A|a]ugust|[S|s]eptember|[O|o]ctober|[n|N]ovember|[d|D]ecember),?\s*(0[1-9]|[1-2][0-9]|3[0-1]),?\s*(19[0-9][0-9]|20[0-9][0-9])", text) return res def format_date(date): res = None if date is not None: month = date[1] try: month = months[month] except(KeyError): pass day = date[2] year = date[3] if len(year) == 2: if int(year[0]) > 2: year = "19" + year else: year = "20" + year res = f"{year}-{month}-{day}" return res if __name__ == "__main__": if len(sys.argv) < 2: raise Exception("Input file not provided") main(sys.argv[1])