s444463/ium_444463
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity
bb5ac435f1
ium_444463/download_data.ipynb

1537 lines
63 KiB
Plaintext
Raw Normal View History

added first homework
2022-03-19 23:50:32 +01:00
{
"cells": [
{
"cell_type": "code",
added Jenkinsfile
2022-03-21 11:00:03 +01:00
"execution_count": 28,
added first homework
2022-03-19 23:50:32 +01:00
"id": "5e2107a5",
"metadata": {},
"outputs": [],
"source": [
"#Skrypt do ściagnięcia zbiory danych\n"
]
},
{
"cell_type": "code",
added Jenkinsfile
2022-03-21 11:00:03 +01:00
"execution_count": 29,
added first homework
2022-03-19 23:50:32 +01:00
"id": "bcc889e5",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
refactoring
2022-03-21 10:06:54 +01:00
"Requirement already satisfied: kaggle in /home/students/s444463/.local/lib/python3.8/site-packages (1.5.12)\n",
added Jenkinsfile
2022-03-21 11:00:03 +01:00
"Requirement already satisfied: tqdm in /home/students/s444463/.local/lib/python3.8/site-packages (from kaggle) (4.63.0)\n",
refactoring
2022-03-21 10:06:54 +01:00
"Requirement already satisfied: urllib3 in /usr/lib/python3/dist-packages (from kaggle) (1.25.8)\n",
refactoring
2022-03-21 10:02:31 +01:00
"Requirement already satisfied: six>=1.10 in /usr/lib/python3/dist-packages (from kaggle) (1.14.0)\n",
added Jenkinsfile
2022-03-21 11:00:03 +01:00
"Requirement already satisfied: certifi in /usr/lib/python3/dist-packages (from kaggle) (2019.11.28)\n",
"Requirement already satisfied: requests in /usr/lib/python3/dist-packages (from kaggle) (2.22.0)\n",
"Requirement already satisfied: python-slugify in /home/students/s444463/.local/lib/python3.8/site-packages (from kaggle) (6.1.1)\n",
refactoring
2022-03-21 10:06:54 +01:00
"Requirement already satisfied: python-dateutil in /usr/lib/python3/dist-packages (from kaggle) (2.7.3)\n",
"Requirement already satisfied: text-unidecode>=1.3 in /home/students/s444463/.local/lib/python3.8/site-packages (from python-slugify->kaggle) (1.3)\n",
refactoring
2022-03-21 10:02:31 +01:00
"\u001b[33mWARNING: You are using pip version 21.2.4; however, version 22.0.4 is available.\n",
added first homework
2022-03-19 23:50:32 +01:00
"You should consider upgrading via the '/usr/bin/python3 -m pip install --upgrade pip' command.\u001b[0m\n",
refactoring
2022-03-21 10:02:31 +01:00
"Requirement already satisfied: pandas in /usr/lib/python3/dist-packages (0.25.3)\n",
"\u001b[33mWARNING: You are using pip version 21.2.4; however, version 22.0.4 is available.\n",
"You should consider upgrading via the '/usr/bin/python3 -m pip install --upgrade pip' command.\u001b[0m\n",
"Requirement already satisfied: numpy in /usr/lib/python3/dist-packages (1.17.4)\n",
"\u001b[33mWARNING: You are using pip version 21.2.4; however, version 22.0.4 is available.\n",
added first homework
2022-03-19 23:50:32 +01:00
"You should consider upgrading via the '/usr/bin/python3 -m pip install --upgrade pip' command.\u001b[0m\n"
]
}
],
"source": [
"!pip install --user kaggle #API Kaggle, do pobrania zbioru\n",
changed Nan to empty strings
2022-03-20 22:09:51 +01:00
"!pip install --user pandas\n",
"!pip install --user numpy"
added first homework
2022-03-19 23:50:32 +01:00
]
},
{
"cell_type": "code",
added Jenkinsfile
2022-03-21 11:00:03 +01:00
"execution_count": 30,
added first homework
2022-03-19 23:50:32 +01:00
"id": "02a4034f",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
refactoring
2022-03-21 10:02:31 +01:00
"/bin/bash: kaggle: command not found\r\n"
added first homework
2022-03-19 23:50:32 +01:00
]
}
],
"source": [
"# Żeby poniższa komenda zadziałała, musisz posiadać plik ~/.kaggle/kaggle.json, zawierający Kaggle API token.\n",
"# Instrukcje: https://www.kaggle.com/docs/api\n",
"!kaggle datasets download -d shivamb/real-or-fake-fake-jobposting-prediction"
]
},
{
"cell_type": "code",
added Jenkinsfile
2022-03-21 11:00:03 +01:00
"execution_count": 31,
added first homework
2022-03-19 23:50:32 +01:00
"id": "5035aef0",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
refactoring
2022-03-21 10:02:31 +01:00
"unzip: cannot find or open real-or-fake-fake-jobposting-prediction.zip, real-or-fake-fake-jobposting-prediction.zip.zip or real-or-fake-fake-jobposting-prediction.zip.ZIP.\r\n"
added first homework
2022-03-19 23:50:32 +01:00
]
}
],
"source": [
"!unzip -o real-or-fake-fake-jobposting-prediction.zip"
]
},
{
"cell_type": "code",
added Jenkinsfile
2022-03-21 11:00:03 +01:00
"execution_count": 32,
added first homework
2022-03-19 23:50:32 +01:00
"id": "14344d2f",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
refactoring
2022-03-21 10:06:54 +01:00
"Requirement already satisfied: seaborn in /home/students/s444463/.local/lib/python3.8/site-packages (0.11.2)\n",
refactoring
2022-03-21 10:02:31 +01:00
"Requirement already satisfied: numpy>=1.15 in /usr/lib/python3/dist-packages (from seaborn) (1.17.4)\n",
refactoring
2022-03-21 10:06:54 +01:00
"Requirement already satisfied: scipy>=1.0 in /usr/lib/python3/dist-packages (from seaborn) (1.3.3)\n",
refactoring
2022-03-21 10:02:31 +01:00
"Requirement already satisfied: matplotlib>=2.2 in /home/students/s444463/.local/lib/python3.8/site-packages (from seaborn) (3.4.3)\n",
added Jenkinsfile
2022-03-21 11:00:03 +01:00
"Requirement already satisfied: pandas>=0.23 in /usr/lib/python3/dist-packages (from seaborn) (0.25.3)\n",
refactoring
2022-03-21 10:02:31 +01:00
"Requirement already satisfied: pillow>=6.2.0 in /usr/local/lib/python3.8/dist-packages (from matplotlib>=2.2->seaborn) (8.3.2)\n",
"Requirement already satisfied: pyparsing>=2.2.1 in /usr/local/lib/python3.8/dist-packages (from matplotlib>=2.2->seaborn) (2.4.7)\n",
added Jenkinsfile
2022-03-21 11:00:03 +01:00
"Requirement already satisfied: cycler>=0.10 in /home/students/s444463/.local/lib/python3.8/site-packages (from matplotlib>=2.2->seaborn) (0.10.0)\n",
refactoring
2022-03-21 10:06:54 +01:00
"Requirement already satisfied: kiwisolver>=1.0.1 in /home/students/s444463/.local/lib/python3.8/site-packages (from matplotlib>=2.2->seaborn) (1.3.2)\n",
added Jenkinsfile
2022-03-21 11:00:03 +01:00
"Requirement already satisfied: python-dateutil>=2.7 in /usr/lib/python3/dist-packages (from matplotlib>=2.2->seaborn) (2.7.3)\n",
refactoring
2022-03-21 10:02:31 +01:00
"Requirement already satisfied: six in /usr/lib/python3/dist-packages (from cycler>=0.10->matplotlib>=2.2->seaborn) (1.14.0)\n",
"\u001b[33mWARNING: You are using pip version 21.2.4; however, version 22.0.4 is available.\n",
added first homework
2022-03-19 23:50:32 +01:00
"You should consider upgrading via the '/usr/bin/python3 -m pip install --upgrade pip' command.\u001b[0m\n"
]
}
],
"source": [
"!pip install --user seaborn"
]
},
{
"cell_type": "code",
added Jenkinsfile
2022-03-21 11:00:03 +01:00
"execution_count": 33,
added first homework
2022-03-19 23:50:32 +01:00
"id": "0f5ebfab",
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>job_id</th>\n",
" <th>title</th>\n",
" <th>location</th>\n",
" <th>department</th>\n",
" <th>salary_range</th>\n",
" <th>company_profile</th>\n",
" <th>description</th>\n",
" <th>requirements</th>\n",
" <th>benefits</th>\n",
" <th>telecommuting</th>\n",
" <th>has_company_logo</th>\n",
" <th>has_questions</th>\n",
" <th>employment_type</th>\n",
" <th>required_experience</th>\n",
" <th>required_education</th>\n",
" <th>industry</th>\n",
" <th>function</th>\n",
" <th>fraudulent</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>1</td>\n",
" <td>Marketing Intern</td>\n",
" <td>US, NY, New York</td>\n",
" <td>Marketing</td>\n",
" <td>NaN</td>\n",
" <td>We're Food52, and we've created a groundbreaki...</td>\n",
" <td>Food52, a fast-growing, James Beard Award-winn...</td>\n",
" <td>Experience with content management systems a m...</td>\n",
" <td>NaN</td>\n",
" <td>0</td>\n",
" <td>1</td>\n",
" <td>0</td>\n",
" <td>Other</td>\n",
" <td>Internship</td>\n",
" <td>NaN</td>\n",
" <td>NaN</td>\n",
" <td>Marketing</td>\n",
" <td>0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>2</td>\n",
" <td>Customer Service - Cloud Video Production</td>\n",
" <td>NZ, , Auckland</td>\n",
" <td>Success</td>\n",
" <td>NaN</td>\n",
" <td>90 Seconds, the worlds Cloud Video Production ...</td>\n",
" <td>Organised - Focused - Vibrant - Awesome!Do you...</td>\n",
" <td>What we expect from you:Your key responsibilit...</td>\n",
" <td>What you will get from usThrough being part of...</td>\n",
" <td>0</td>\n",
" <td>1</td>\n",
" <td>0</td>\n",
" <td>Full-time</td>\n",
" <td>Not Applicable</td>\n",
" <td>NaN</td>\n",
" <td>Marketing and Advertising</td>\n",
" <td>Customer Service</td>\n",
" <td>0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>3</td>\n",
" <td>Commissioning Machinery Assistant (CMA)</td>\n",
" <td>US, IA, Wever</td>\n",
" <td>NaN</td>\n",
" <td>NaN</td>\n",
" <td>Valor Services provides Workforce Solutions th...</td>\n",
" <td>Our client, located in Houston, is actively se...</td>\n",
" <td>Implement pre-commissioning and commissioning ...</td>\n",
" <td>NaN</td>\n",
" <td>0</td>\n",
" <td>1</td>\n",
" <td>0</td>\n",
" <td>NaN</td>\n",
" <td>NaN</td>\n",
" <td>NaN</td>\n",
" <td>NaN</td>\n",
" <td>NaN</td>\n",
" <td>0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>4</td>\n",
" <td>Account Executive - Washington DC</td>\n",
" <td>US, DC, Washington</td>\n",
" <td>Sales</td>\n",
" <td>NaN</td>\n",
" <td>Our passion for improving quality of life thro...</td>\n",
" <td>THE COMPANY: ESRI – Environmental Systems Rese...</td>\n",
" <td>EDUCATION: Bachelor’s or Master’s in GIS, busi...</td>\n",
" <td>Our culture is anything but corporate—we have ...</td>\n",
" <td>0</td>\n",
" <td>1</td>\n",
" <td>0</td>\n",
" <td>Full-time</td>\n",
" <td>Mid-Senior level</td>\n",
" <td>Bachelor's Degree</td>\n",
" <td>Computer Software</td>\n",
" <td>Sales</td>\n",
" <td>0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>5</td>\n",
" <td>Bill Review Manager</td>\n",
" <td>US, FL, Fort Worth</td>\n",
" <td>NaN</td>\n",
" <td>NaN</td>\n",
" <td>SpotSource Solutions LLC is a Global Human Cap...</td>\n",
" <td>JOB TITLE: Itemization Review ManagerLOCATION:...</td>\n",
" <td>QUALIFICATIONS:RN license in the State of Texa...</td>\n",
" <td>Full Benefits Offered</td>\n",
" <td>0</td>\n",
" <td>1</td>\n",
" <td>1</td>\n",
" <td>Full-time</td>\n",
" <td>Mid-Senior level</td>\n",
" <td>Bachelor's Degree</td>\n",
" <td>Hospital &amp; Health Care</td>\n",
" <td>Health Care Provider</td>\n",
" <td>0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>...</th>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>17875</th>\n",
" <td>17876</td>\n",
" <td>Account Director - Distribution</td>\n",
" <td>CA, ON, Toronto</td>\n",
" <td>Sales</td>\n",
" <td>NaN</td>\n",
" <td>Vend is looking for some awesome new talent to...</td>\n",
" <td>Just in case this is the first time you’ve vis...</td>\n",
" <td>To ace this role you:Will eat comprehensive St...</td>\n",
" <td>What can you expect from us?We have an open cu...</td>\n",
" <td>0</td>\n",
" <td>1</td>\n",
" <td>1</td>\n",
" <td>Full-time</td>\n",
" <td>Mid-Senior level</td>\n",
" <td>NaN</td>\n",
" <td>Computer Software</td>\n",
" <td>Sales</td>\n",
" <td>0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>17876</th>\n",
" <td>17877</td>\n",
" <td>Payroll Accountant</td>\n",
" <td>US, PA, Philadelphia</td>\n",
" <td>Accounting</td>\n",
" <td>NaN</td>\n",
" <td>WebLinc is the e-commerce platform and service...</td>\n",
" <td>The Payroll Accountant will focus primarily on...</td>\n",
" <td>- B.A. or B.S. in Accounting- Desire to have f...</td>\n",
" <td>Health &amp;amp; WellnessMedical planPrescription ...</td>\n",
" <td>0</td>\n",
" <td>1</td>\n",
" <td>1</td>\n",
" <td>Full-time</td>\n",
" <td>Mid-Senior level</td>\n",
" <td>Bachelor's Degree</td>\n",
" <td>Internet</td>\n",
" <td>Accounting/Auditing</td>\n",
" <td>0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>17877</th>\n",
" <td>17878</td>\n",
" <td>Project Cost Control Staff Engineer - Cost Con...</td>\n",
" <td>US, TX, Houston</td>\n",
" <td>NaN</td>\n",
" <td>NaN</td>\n",
" <td>We Provide Full Time Permanent Positions for m...</td>\n",
" <td>Experienced Project Cost Control Staff Enginee...</td>\n",
" <td>At least 12 years professional experience.Abil...</td>\n",
" <td>NaN</td>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" <td>Full-time</td>\n",
" <td>NaN</td>\n",
" <td>NaN</td>\n",
" <td>NaN</td>\n",
" <td>NaN</td>\n",
" <td>0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>17878</th>\n",
" <td>17879</td>\n",
" <td>Graphic Designer</td>\n",
" <td>NG, LA, Lagos</td>\n",
" <td>NaN</td>\n",
" <td>NaN</td>\n",
" <td>NaN</td>\n",
" <td>Nemsia Studios is looking for an experienced v...</td>\n",
" <td>1. Must be fluent in the latest versions of Co...</td>\n",
" <td>Competitive salary (compensation will be based...</td>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" <td>1</td>\n",
" <td>Contract</td>\n",
" <td>Not Applicable</td>\n",
" <td>Professional</td>\n",
" <td>Graphic Design</td>\n",
" <td>Design</td>\n",
" <td>0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>17879</th>\n",
" <td>17880</td>\n",
" <td>Web Application Developers</td>\n",
" <td>NZ, N, Wellington</td>\n",
" <td>Engineering</td>\n",
" <td>NaN</td>\n",
" <td>Vend is looking for some awesome new talent to...</td>\n",
" <td>Who are we?Vend is an award winning web based ...</td>\n",
" <td>We want to hear from you if:You have an in-dep...</td>\n",
" <td>NaN</td>\n",
" <td>0</td>\n",
" <td>1</td>\n",
" <td>1</td>\n",
" <td>Full-time</td>\n",
" <td>Mid-Senior level</td>\n",
" <td>NaN</td>\n",
" <td>Computer Software</td>\n",
" <td>Engineering</td>\n",
" <td>0</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"<p>17880 rows × 18 columns</p>\n",
"</div>"
],
"text/plain": [
" job_id title \\\n",
"0 1 Marketing Intern \n",
"1 2 Customer Service - Cloud Video Production \n",
"2 3 Commissioning Machinery Assistant (CMA) \n",
"3 4 Account Executive - Washington DC \n",
"4 5 Bill Review Manager \n",
"... ... ... \n",
"17875 17876 Account Director - Distribution \n",
"17876 17877 Payroll Accountant \n",
"17877 17878 Project Cost Control Staff Engineer - Cost Con... \n",
"17878 17879 Graphic Designer \n",
"17879 17880 Web Application Developers \n",
"\n",
" location department salary_range \\\n",
"0 US, NY, New York Marketing NaN \n",
"1 NZ, , Auckland Success NaN \n",
"2 US, IA, Wever NaN NaN \n",
"3 US, DC, Washington Sales NaN \n",
"4 US, FL, Fort Worth NaN NaN \n",
"... ... ... ... \n",
"17875 CA, ON, Toronto Sales NaN \n",
"17876 US, PA, Philadelphia Accounting NaN \n",
"17877 US, TX, Houston NaN NaN \n",
"17878 NG, LA, Lagos NaN NaN \n",
"17879 NZ, N, Wellington Engineering NaN \n",
"\n",
" company_profile \\\n",
"0 We're Food52, and we've created a groundbreaki... \n",
"1 90 Seconds, the worlds Cloud Video Production ... \n",
"2 Valor Services provides Workforce Solutions th... \n",
"3 Our passion for improving quality of life thro... \n",
"4 SpotSource Solutions LLC is a Global Human Cap... \n",
"... ... \n",
"17875 Vend is looking for some awesome new talent to... \n",
"17876 WebLinc is the e-commerce platform and service... \n",
"17877 We Provide Full Time Permanent Positions for m... \n",
"17878 NaN \n",
"17879 Vend is looking for some awesome new talent to... \n",
"\n",
" description \\\n",
"0 Food52, a fast-growing, James Beard Award-winn... \n",
"1 Organised - Focused - Vibrant - Awesome!Do you... \n",
"2 Our client, located in Houston, is actively se... \n",
"3 THE COMPANY: ESRI – Environmental Systems Rese... \n",
"4 JOB TITLE: Itemization Review ManagerLOCATION:... \n",
"... ... \n",
"17875 Just in case this is the first time you’ve vis... \n",
"17876 The Payroll Accountant will focus primarily on... \n",
"17877 Experienced Project Cost Control Staff Enginee... \n",
"17878 Nemsia Studios is looking for an experienced v... \n",
"17879 Who are we?Vend is an award winning web based ... \n",
"\n",
" requirements \\\n",
"0 Experience with content management systems a m... \n",
"1 What we expect from you:Your key responsibilit... \n",
"2 Implement pre-commissioning and commissioning ... \n",
"3 EDUCATION: Bachelor’s or Master’s in GIS, busi... \n",
"4 QUALIFICATIONS:RN license in the State of Texa... \n",
"... ... \n",
"17875 To ace this role you:Will eat comprehensive St... \n",
"17876 - B.A. or B.S. in Accounting- Desire to have f... \n",
"17877 At least 12 years professional experience.Abil... \n",
"17878 1. Must be fluent in the latest versions of Co... \n",
"17879 We want to hear from you if:You have an in-dep... \n",
"\n",
" benefits telecommuting \\\n",
"0 NaN 0 \n",
"1 What you will get from usThrough being part of... 0 \n",
"2 NaN 0 \n",
"3 Our culture is anything but corporate—we have ... 0 \n",
"4 Full Benefits Offered 0 \n",
"... ... ... \n",
"17875 What can you expect from us?We have an open cu... 0 \n",
"17876 Health &amp; WellnessMedical planPrescription ... 0 \n",
"17877 NaN 0 \n",
"17878 Competitive salary (compensation will be based... 0 \n",
"17879 NaN 0 \n",
"\n",
" has_company_logo has_questions employment_type required_experience \\\n",
"0 1 0 Other Internship \n",
"1 1 0 Full-time Not Applicable \n",
"2 1 0 NaN NaN \n",
"3 1 0 Full-time Mid-Senior level \n",
"4 1 1 Full-time Mid-Senior level \n",
"... ... ... ... ... \n",
"17875 1 1 Full-time Mid-Senior level \n",
"17876 1 1 Full-time Mid-Senior level \n",
"17877 0 0 Full-time NaN \n",
"17878 0 1 Contract Not Applicable \n",
"17879 1 1 Full-time Mid-Senior level \n",
"\n",
" required_education industry function \\\n",
"0 NaN NaN Marketing \n",
"1 NaN Marketing and Advertising Customer Service \n",
"2 NaN NaN NaN \n",
"3 Bachelor's Degree Computer Software Sales \n",
"4 Bachelor's Degree Hospital & Health Care Health Care Provider \n",
"... ... ... ... \n",
"17875 NaN Computer Software Sales \n",
"17876 Bachelor's Degree Internet Accounting/Auditing \n",
"17877 NaN NaN NaN \n",
"17878 Professional Graphic Design Design \n",
"17879 NaN Computer Software Engineering \n",
"\n",
" fraudulent \n",
"0 0 \n",
"1 0 \n",
"2 0 \n",
"3 0 \n",
"4 0 \n",
"... ... \n",
"17875 0 \n",
"17876 0 \n",
"17877 0 \n",
"17878 0 \n",
"17879 0 \n",
"\n",
"[17880 rows x 18 columns]"
]
},
added Jenkinsfile
2022-03-21 11:00:03 +01:00
"execution_count": 33,
added first homework
2022-03-19 23:50:32 +01:00
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"import pandas as pd\n",
"data=pd.read_csv('fake_job_postings.csv')\n",
"data\n"
]
},
{
"cell_type": "code",
added Jenkinsfile
2022-03-21 11:00:03 +01:00
"execution_count": 34,
refactored
2022-03-19 23:57:55 +01:00
"id": "edbf49da",
added first homework
2022-03-19 23:50:32 +01:00
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
refactored
2022-03-19 23:57:55 +01:00
"17880 fake_job_postings.csv\r\n"
added first homework
2022-03-19 23:50:32 +01:00
]
}
],
"source": [
"#Wielkosc zbioru\n",
refactored
2022-03-19 23:57:55 +01:00
"!wc -l fake_job_postings.csv"
added first homework
2022-03-19 23:50:32 +01:00
]
},
{
"cell_type": "code",
added Jenkinsfile
2022-03-21 11:00:03 +01:00
"execution_count": 35,
refactoring
2022-03-21 10:06:54 +01:00
"id": "e60b3f32",
added first homework
2022-03-19 23:50:32 +01:00
"metadata": {},
"outputs": [
{
refactoring
2022-03-21 10:06:54 +01:00
"data": {
"text/plain": [
"Sales 551\n",
"Engineering 487\n",
"Marketing 401\n",
"Operations 270\n",
"IT 225\n",
" ... \n",
"Capoo 1\n",
"Engineering - Hardware 1\n",
"Utilities 1\n",
"i 1\n",
"TECH 1\n",
"Name: department, Length: 1337, dtype: int64"
]
},
added Jenkinsfile
2022-03-21 11:00:03 +01:00
"execution_count": 35,
refactoring
2022-03-21 10:06:54 +01:00
"metadata": {},
"output_type": "execute_result"
added first homework
2022-03-19 23:50:32 +01:00
}
],
"source": [
refactoring
2022-03-21 10:06:54 +01:00
"data[\"department\"].value_counts()"
added first homework
2022-03-19 23:50:32 +01:00
]
},
{
"cell_type": "code",
added Jenkinsfile
2022-03-21 11:00:03 +01:00
"execution_count": 36,
"id": "ddb2fc38",
added first homework
2022-03-19 23:50:32 +01:00
"metadata": {},
added Jenkinsfile
2022-03-21 11:00:03 +01:00
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>job_id</th>\n",
" <th>title</th>\n",
" <th>location</th>\n",
" <th>department</th>\n",
" <th>salary_range</th>\n",
" <th>company_profile</th>\n",
" <th>description</th>\n",
" <th>requirements</th>\n",
" <th>benefits</th>\n",
" <th>telecommuting</th>\n",
" <th>has_company_logo</th>\n",
" <th>has_questions</th>\n",
" <th>employment_type</th>\n",
" <th>required_experience</th>\n",
" <th>required_education</th>\n",
" <th>industry</th>\n",
" <th>function</th>\n",
" <th>fraudulent</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>1</td>\n",
" <td>Marketing Intern</td>\n",
" <td>US, NY, New York</td>\n",
" <td>Marketing</td>\n",
" <td></td>\n",
" <td>We're Food52, and we've created a groundbreaki...</td>\n",
" <td>Food52, a fast-growing, James Beard Award-winn...</td>\n",
" <td>Experience with content management systems a m...</td>\n",
" <td></td>\n",
" <td>0</td>\n",
" <td>1</td>\n",
" <td>0</td>\n",
" <td>Other</td>\n",
" <td>Internship</td>\n",
" <td></td>\n",
" <td></td>\n",
" <td>Marketing</td>\n",
" <td>0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>2</td>\n",
" <td>Customer Service - Cloud Video Production</td>\n",
" <td>NZ, , Auckland</td>\n",
" <td>Success</td>\n",
" <td></td>\n",
" <td>90 Seconds, the worlds Cloud Video Production ...</td>\n",
" <td>Organised - Focused - Vibrant - Awesome!Do you...</td>\n",
" <td>What we expect from you:Your key responsibilit...</td>\n",
" <td>What you will get from usThrough being part of...</td>\n",
" <td>0</td>\n",
" <td>1</td>\n",
" <td>0</td>\n",
" <td>Full-time</td>\n",
" <td>Not Applicable</td>\n",
" <td></td>\n",
" <td>Marketing and Advertising</td>\n",
" <td>Customer Service</td>\n",
" <td>0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>3</td>\n",
" <td>Commissioning Machinery Assistant (CMA)</td>\n",
" <td>US, IA, Wever</td>\n",
" <td></td>\n",
" <td></td>\n",
" <td>Valor Services provides Workforce Solutions th...</td>\n",
" <td>Our client, located in Houston, is actively se...</td>\n",
" <td>Implement pre-commissioning and commissioning ...</td>\n",
" <td></td>\n",
" <td>0</td>\n",
" <td>1</td>\n",
" <td>0</td>\n",
" <td></td>\n",
" <td></td>\n",
" <td></td>\n",
" <td></td>\n",
" <td></td>\n",
" <td>0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>4</td>\n",
" <td>Account Executive - Washington DC</td>\n",
" <td>US, DC, Washington</td>\n",
" <td>Sales</td>\n",
" <td></td>\n",
" <td>Our passion for improving quality of life thro...</td>\n",
" <td>THE COMPANY: ESRI – Environmental Systems Rese...</td>\n",
" <td>EDUCATION: Bachelor’s or Master’s in GIS, busi...</td>\n",
" <td>Our culture is anything but corporate—we have ...</td>\n",
" <td>0</td>\n",
" <td>1</td>\n",
" <td>0</td>\n",
" <td>Full-time</td>\n",
" <td>Mid-Senior level</td>\n",
" <td>Bachelor's Degree</td>\n",
" <td>Computer Software</td>\n",
" <td>Sales</td>\n",
" <td>0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>5</td>\n",
" <td>Bill Review Manager</td>\n",
" <td>US, FL, Fort Worth</td>\n",
" <td></td>\n",
" <td></td>\n",
" <td>SpotSource Solutions LLC is a Global Human Cap...</td>\n",
" <td>JOB TITLE: Itemization Review ManagerLOCATION:...</td>\n",
" <td>QUALIFICATIONS:RN license in the State of Texa...</td>\n",
" <td>Full Benefits Offered</td>\n",
" <td>0</td>\n",
" <td>1</td>\n",
" <td>1</td>\n",
" <td>Full-time</td>\n",
" <td>Mid-Senior level</td>\n",
" <td>Bachelor's Degree</td>\n",
" <td>Hospital &amp; Health Care</td>\n",
" <td>Health Care Provider</td>\n",
" <td>0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>...</th>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>17875</th>\n",
" <td>17876</td>\n",
" <td>Account Director - Distribution</td>\n",
" <td>CA, ON, Toronto</td>\n",
" <td>Sales</td>\n",
" <td></td>\n",
" <td>Vend is looking for some awesome new talent to...</td>\n",
" <td>Just in case this is the first time you’ve vis...</td>\n",
" <td>To ace this role you:Will eat comprehensive St...</td>\n",
" <td>What can you expect from us?We have an open cu...</td>\n",
" <td>0</td>\n",
" <td>1</td>\n",
" <td>1</td>\n",
" <td>Full-time</td>\n",
" <td>Mid-Senior level</td>\n",
" <td></td>\n",
" <td>Computer Software</td>\n",
" <td>Sales</td>\n",
" <td>0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>17876</th>\n",
" <td>17877</td>\n",
" <td>Payroll Accountant</td>\n",
" <td>US, PA, Philadelphia</td>\n",
" <td>Accounting</td>\n",
" <td></td>\n",
" <td>WebLinc is the e-commerce platform and service...</td>\n",
" <td>The Payroll Accountant will focus primarily on...</td>\n",
" <td>- B.A. or B.S. in Accounting- Desire to have f...</td>\n",
" <td>Health &amp;amp; WellnessMedical planPrescription ...</td>\n",
" <td>0</td>\n",
" <td>1</td>\n",
" <td>1</td>\n",
" <td>Full-time</td>\n",
" <td>Mid-Senior level</td>\n",
" <td>Bachelor's Degree</td>\n",
" <td>Internet</td>\n",
" <td>Accounting/Auditing</td>\n",
" <td>0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>17877</th>\n",
" <td>17878</td>\n",
" <td>Project Cost Control Staff Engineer - Cost Con...</td>\n",
" <td>US, TX, Houston</td>\n",
" <td></td>\n",
" <td></td>\n",
" <td>We Provide Full Time Permanent Positions for m...</td>\n",
" <td>Experienced Project Cost Control Staff Enginee...</td>\n",
" <td>At least 12 years professional experience.Abil...</td>\n",
" <td></td>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" <td>Full-time</td>\n",
" <td></td>\n",
" <td></td>\n",
" <td></td>\n",
" <td></td>\n",
" <td>0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>17878</th>\n",
" <td>17879</td>\n",
" <td>Graphic Designer</td>\n",
" <td>NG, LA, Lagos</td>\n",
" <td></td>\n",
" <td></td>\n",
" <td></td>\n",
" <td>Nemsia Studios is looking for an experienced v...</td>\n",
" <td>1. Must be fluent in the latest versions of Co...</td>\n",
" <td>Competitive salary (compensation will be based...</td>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" <td>1</td>\n",
" <td>Contract</td>\n",
" <td>Not Applicable</td>\n",
" <td>Professional</td>\n",
" <td>Graphic Design</td>\n",
" <td>Design</td>\n",
" <td>0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>17879</th>\n",
" <td>17880</td>\n",
" <td>Web Application Developers</td>\n",
" <td>NZ, N, Wellington</td>\n",
" <td>Engineering</td>\n",
" <td></td>\n",
" <td>Vend is looking for some awesome new talent to...</td>\n",
" <td>Who are we?Vend is an award winning web based ...</td>\n",
" <td>We want to hear from you if:You have an in-dep...</td>\n",
" <td></td>\n",
" <td>0</td>\n",
" <td>1</td>\n",
" <td>1</td>\n",
" <td>Full-time</td>\n",
" <td>Mid-Senior level</td>\n",
" <td></td>\n",
" <td>Computer Software</td>\n",
" <td>Engineering</td>\n",
" <td>0</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"<p>17880 rows × 18 columns</p>\n",
"</div>"
],
"text/plain": [
" job_id title \\\n",
"0 1 Marketing Intern \n",
"1 2 Customer Service - Cloud Video Production \n",
"2 3 Commissioning Machinery Assistant (CMA) \n",
"3 4 Account Executive - Washington DC \n",
"4 5 Bill Review Manager \n",
"... ... ... \n",
"17875 17876 Account Director - Distribution \n",
"17876 17877 Payroll Accountant \n",
"17877 17878 Project Cost Control Staff Engineer - Cost Con... \n",
"17878 17879 Graphic Designer \n",
"17879 17880 Web Application Developers \n",
"\n",
" location department salary_range \\\n",
"0 US, NY, New York Marketing \n",
"1 NZ, , Auckland Success \n",
"2 US, IA, Wever \n",
"3 US, DC, Washington Sales \n",
"4 US, FL, Fort Worth \n",
"... ... ... ... \n",
"17875 CA, ON, Toronto Sales \n",
"17876 US, PA, Philadelphia Accounting \n",
"17877 US, TX, Houston \n",
"17878 NG, LA, Lagos \n",
"17879 NZ, N, Wellington Engineering \n",
"\n",
" company_profile \\\n",
"0 We're Food52, and we've created a groundbreaki... \n",
"1 90 Seconds, the worlds Cloud Video Production ... \n",
"2 Valor Services provides Workforce Solutions th... \n",
"3 Our passion for improving quality of life thro... \n",
"4 SpotSource Solutions LLC is a Global Human Cap... \n",
"... ... \n",
"17875 Vend is looking for some awesome new talent to... \n",
"17876 WebLinc is the e-commerce platform and service... \n",
"17877 We Provide Full Time Permanent Positions for m... \n",
"17878 \n",
"17879 Vend is looking for some awesome new talent to... \n",
"\n",
" description \\\n",
"0 Food52, a fast-growing, James Beard Award-winn... \n",
"1 Organised - Focused - Vibrant - Awesome!Do you... \n",
"2 Our client, located in Houston, is actively se... \n",
"3 THE COMPANY: ESRI – Environmental Systems Rese... \n",
"4 JOB TITLE: Itemization Review ManagerLOCATION:... \n",
"... ... \n",
"17875 Just in case this is the first time you’ve vis... \n",
"17876 The Payroll Accountant will focus primarily on... \n",
"17877 Experienced Project Cost Control Staff Enginee... \n",
"17878 Nemsia Studios is looking for an experienced v... \n",
"17879 Who are we?Vend is an award winning web based ... \n",
"\n",
" requirements \\\n",
"0 Experience with content management systems a m... \n",
"1 What we expect from you:Your key responsibilit... \n",
"2 Implement pre-commissioning and commissioning ... \n",
"3 EDUCATION: Bachelor’s or Master’s in GIS, busi... \n",
"4 QUALIFICATIONS:RN license in the State of Texa... \n",
"... ... \n",
"17875 To ace this role you:Will eat comprehensive St... \n",
"17876 - B.A. or B.S. in Accounting- Desire to have f... \n",
"17877 At least 12 years professional experience.Abil... \n",
"17878 1. Must be fluent in the latest versions of Co... \n",
"17879 We want to hear from you if:You have an in-dep... \n",
"\n",
" benefits telecommuting \\\n",
"0 0 \n",
"1 What you will get from usThrough being part of... 0 \n",
"2 0 \n",
"3 Our culture is anything but corporate—we have ... 0 \n",
"4 Full Benefits Offered 0 \n",
"... ... ... \n",
"17875 What can you expect from us?We have an open cu... 0 \n",
"17876 Health &amp; WellnessMedical planPrescription ... 0 \n",
"17877 0 \n",
"17878 Competitive salary (compensation will be based... 0 \n",
"17879 0 \n",
"\n",
" has_company_logo has_questions employment_type required_experience \\\n",
"0 1 0 Other Internship \n",
"1 1 0 Full-time Not Applicable \n",
"2 1 0 \n",
"3 1 0 Full-time Mid-Senior level \n",
"4 1 1 Full-time Mid-Senior level \n",
"... ... ... ... ... \n",
"17875 1 1 Full-time Mid-Senior level \n",
"17876 1 1 Full-time Mid-Senior level \n",
"17877 0 0 Full-time \n",
"17878 0 1 Contract Not Applicable \n",
"17879 1 1 Full-time Mid-Senior level \n",
"\n",
" required_education industry function \\\n",
"0 Marketing \n",
"1 Marketing and Advertising Customer Service \n",
"2 \n",
"3 Bachelor's Degree Computer Software Sales \n",
"4 Bachelor's Degree Hospital & Health Care Health Care Provider \n",
"... ... ... ... \n",
"17875 Computer Software Sales \n",
"17876 Bachelor's Degree Internet Accounting/Auditing \n",
"17877 \n",
"17878 Professional Graphic Design Design \n",
"17879 Computer Software Engineering \n",
"\n",
" fraudulent \n",
"0 0 \n",
"1 0 \n",
"2 0 \n",
"3 0 \n",
"4 0 \n",
"... ... \n",
"17875 0 \n",
"17876 0 \n",
"17877 0 \n",
"17878 0 \n",
"17879 0 \n",
"\n",
"[17880 rows x 18 columns]"
]
},
"execution_count": 36,
"metadata": {},
"output_type": "execute_result"
}
],
added first homework
2022-03-19 23:50:32 +01:00
"source": [
refactoring
2022-03-21 10:06:54 +01:00
"import numpy as np\n",
"data = data.replace(np.nan, '', regex=True)\n",
"data"
added first homework
2022-03-19 23:50:32 +01:00
]
},
{
"cell_type": "code",
added Jenkinsfile
2022-03-21 11:00:03 +01:00
"execution_count": 37,
refactored
2022-03-19 23:57:55 +01:00
"id": "c5ac75f5",
added first homework
2022-03-19 23:50:32 +01:00
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>job_id</th>\n",
" <th>title</th>\n",
" <th>location</th>\n",
" <th>department</th>\n",
" <th>salary_range</th>\n",
" <th>company_profile</th>\n",
" <th>description</th>\n",
" <th>requirements</th>\n",
" <th>benefits</th>\n",
" <th>telecommuting</th>\n",
" <th>has_company_logo</th>\n",
" <th>has_questions</th>\n",
" <th>employment_type</th>\n",
" <th>required_experience</th>\n",
" <th>required_education</th>\n",
" <th>industry</th>\n",
" <th>function</th>\n",
" <th>fraudulent</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>count</th>\n",
" <td>17880.000000</td>\n",
" <td>17880</td>\n",
changed Nan to empty strings
2022-03-20 22:09:51 +01:00
" <td>17880</td>\n",
" <td>17880</td>\n",
" <td>17880</td>\n",
" <td>17880</td>\n",
" <td>17880</td>\n",
" <td>17880</td>\n",
" <td>17880</td>\n",
added first homework
2022-03-19 23:50:32 +01:00
" <td>17880.000000</td>\n",
" <td>17880.000000</td>\n",
" <td>17880.000000</td>\n",
changed Nan to empty strings
2022-03-20 22:09:51 +01:00
" <td>17880</td>\n",
" <td>17880</td>\n",
" <td>17880</td>\n",
" <td>17880</td>\n",
" <td>17880</td>\n",
added first homework
2022-03-19 23:50:32 +01:00
" <td>17880.000000</td>\n",
" </tr>\n",
" <tr>\n",
" <th>unique</th>\n",
" <td>NaN</td>\n",
" <td>11231</td>\n",
changed Nan to empty strings
2022-03-20 22:09:51 +01:00
" <td>3106</td>\n",
" <td>1338</td>\n",
" <td>875</td>\n",
" <td>1710</td>\n",
" <td>14802</td>\n",
" <td>11969</td>\n",
" <td>6206</td>\n",
added first homework
2022-03-19 23:50:32 +01:00
" <td>NaN</td>\n",
" <td>NaN</td>\n",
" <td>NaN</td>\n",
changed Nan to empty strings
2022-03-20 22:09:51 +01:00
" <td>6</td>\n",
" <td>8</td>\n",
" <td>14</td>\n",
" <td>132</td>\n",
" <td>38</td>\n",
added first homework
2022-03-19 23:50:32 +01:00
" <td>NaN</td>\n",
" </tr>\n",
" <tr>\n",
" <th>top</th>\n",
" <td>NaN</td>\n",
" <td>English Teacher Abroad</td>\n",
" <td>GB, LND, London</td>\n",
changed Nan to empty strings
2022-03-20 22:09:51 +01:00
" <td></td>\n",
" <td></td>\n",
" <td></td>\n",
added first homework
2022-03-19 23:50:32 +01:00
" <td>Play with kids, get paid for it Love travel? J...</td>\n",
changed Nan to empty strings
2022-03-20 22:09:51 +01:00
" <td></td>\n",
" <td></td>\n",
added first homework
2022-03-19 23:50:32 +01:00
" <td>NaN</td>\n",
" <td>NaN</td>\n",
" <td>NaN</td>\n",
" <td>Full-time</td>\n",
changed Nan to empty strings
2022-03-20 22:09:51 +01:00
" <td></td>\n",
" <td></td>\n",
" <td></td>\n",
" <td></td>\n",
added first homework
2022-03-19 23:50:32 +01:00
" <td>NaN</td>\n",
" </tr>\n",
" <tr>\n",
" <th>freq</th>\n",
" <td>NaN</td>\n",
" <td>311</td>\n",
" <td>718</td>\n",
changed Nan to empty strings
2022-03-20 22:09:51 +01:00
" <td>11547</td>\n",
" <td>15012</td>\n",
" <td>3308</td>\n",
added first homework
2022-03-19 23:50:32 +01:00
" <td>379</td>\n",
changed Nan to empty strings
2022-03-20 22:09:51 +01:00
" <td>2695</td>\n",
" <td>7210</td>\n",
added first homework
2022-03-19 23:50:32 +01:00
" <td>NaN</td>\n",
" <td>NaN</td>\n",
" <td>NaN</td>\n",
" <td>11620</td>\n",
changed Nan to empty strings
2022-03-20 22:09:51 +01:00
" <td>7050</td>\n",
" <td>8105</td>\n",
" <td>4903</td>\n",
" <td>6455</td>\n",
added first homework
2022-03-19 23:50:32 +01:00
" <td>NaN</td>\n",
" </tr>\n",
" <tr>\n",
" <th>mean</th>\n",
" <td>8940.500000</td>\n",
" <td>NaN</td>\n",
" <td>NaN</td>\n",
" <td>NaN</td>\n",
" <td>NaN</td>\n",
" <td>NaN</td>\n",
" <td>NaN</td>\n",
" <td>NaN</td>\n",
" <td>NaN</td>\n",
" <td>0.042897</td>\n",
" <td>0.795302</td>\n",
" <td>0.491723</td>\n",
" <td>NaN</td>\n",
" <td>NaN</td>\n",
" <td>NaN</td>\n",
" <td>NaN</td>\n",
" <td>NaN</td>\n",
" <td>0.048434</td>\n",
" </tr>\n",
" <tr>\n",
" <th>std</th>\n",
" <td>5161.655742</td>\n",
" <td>NaN</td>\n",
" <td>NaN</td>\n",
" <td>NaN</td>\n",
" <td>NaN</td>\n",
" <td>NaN</td>\n",
" <td>NaN</td>\n",
" <td>NaN</td>\n",
" <td>NaN</td>\n",
" <td>0.202631</td>\n",
" <td>0.403492</td>\n",
" <td>0.499945</td>\n",
" <td>NaN</td>\n",
" <td>NaN</td>\n",
" <td>NaN</td>\n",
" <td>NaN</td>\n",
" <td>NaN</td>\n",
" <td>0.214688</td>\n",
" </tr>\n",
" <tr>\n",
" <th>min</th>\n",
" <td>1.000000</td>\n",
" <td>NaN</td>\n",
" <td>NaN</td>\n",
" <td>NaN</td>\n",
" <td>NaN</td>\n",
" <td>NaN</td>\n",
" <td>NaN</td>\n",
" <td>NaN</td>\n",
" <td>NaN</td>\n",
" <td>0.000000</td>\n",
" <td>0.000000</td>\n",
" <td>0.000000</td>\n",
" <td>NaN</td>\n",
" <td>NaN</td>\n",
" <td>NaN</td>\n",
" <td>NaN</td>\n",
" <td>NaN</td>\n",
" <td>0.000000</td>\n",
" </tr>\n",
" <tr>\n",
" <th>25%</th>\n",
" <td>4470.750000</td>\n",
" <td>NaN</td>\n",
" <td>NaN</td>\n",
" <td>NaN</td>\n",
" <td>NaN</td>\n",
" <td>NaN</td>\n",
" <td>NaN</td>\n",
" <td>NaN</td>\n",
" <td>NaN</td>\n",
" <td>0.000000</td>\n",
" <td>1.000000</td>\n",
" <td>0.000000</td>\n",
" <td>NaN</td>\n",
" <td>NaN</td>\n",
" <td>NaN</td>\n",
" <td>NaN</td>\n",
" <td>NaN</td>\n",
" <td>0.000000</td>\n",
" </tr>\n",
" <tr>\n",
" <th>50%</th>\n",
" <td>8940.500000</td>\n",
" <td>NaN</td>\n",
" <td>NaN</td>\n",
" <td>NaN</td>\n",
" <td>NaN</td>\n",
" <td>NaN</td>\n",
" <td>NaN</td>\n",
" <td>NaN</td>\n",
" <td>NaN</td>\n",
" <td>0.000000</td>\n",
" <td>1.000000</td>\n",
" <td>0.000000</td>\n",
" <td>NaN</td>\n",
" <td>NaN</td>\n",
" <td>NaN</td>\n",
" <td>NaN</td>\n",
" <td>NaN</td>\n",
" <td>0.000000</td>\n",
" </tr>\n",
" <tr>\n",
" <th>75%</th>\n",
" <td>13410.250000</td>\n",
" <td>NaN</td>\n",
" <td>NaN</td>\n",
" <td>NaN</td>\n",
" <td>NaN</td>\n",
" <td>NaN</td>\n",
" <td>NaN</td>\n",
" <td>NaN</td>\n",
" <td>NaN</td>\n",
" <td>0.000000</td>\n",
" <td>1.000000</td>\n",
" <td>1.000000</td>\n",
" <td>NaN</td>\n",
" <td>NaN</td>\n",
" <td>NaN</td>\n",
" <td>NaN</td>\n",
" <td>NaN</td>\n",
" <td>0.000000</td>\n",
" </tr>\n",
" <tr>\n",
" <th>max</th>\n",
" <td>17880.000000</td>\n",
" <td>NaN</td>\n",
" <td>NaN</td>\n",
" <td>NaN</td>\n",
" <td>NaN</td>\n",
" <td>NaN</td>\n",
" <td>NaN</td>\n",
" <td>NaN</td>\n",
" <td>NaN</td>\n",
" <td>1.000000</td>\n",
" <td>1.000000</td>\n",
" <td>1.000000</td>\n",
" <td>NaN</td>\n",
" <td>NaN</td>\n",
" <td>NaN</td>\n",
" <td>NaN</td>\n",
" <td>NaN</td>\n",
" <td>1.000000</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" job_id title location department \\\n",
changed Nan to empty strings
2022-03-20 22:09:51 +01:00
"count 17880.000000 17880 17880 17880 \n",
"unique NaN 11231 3106 1338 \n",
"top NaN English Teacher Abroad GB, LND, London \n",
"freq NaN 311 718 11547 \n",
added first homework
2022-03-19 23:50:32 +01:00
"mean 8940.500000 NaN NaN NaN \n",
"std 5161.655742 NaN NaN NaN \n",
"min 1.000000 NaN NaN NaN \n",
"25% 4470.750000 NaN NaN NaN \n",
"50% 8940.500000 NaN NaN NaN \n",
"75% 13410.250000 NaN NaN NaN \n",
"max 17880.000000 NaN NaN NaN \n",
"\n",
changed Nan to empty strings
2022-03-20 22:09:51 +01:00
" salary_range company_profile \\\n",
"count 17880 17880 \n",
"unique 875 1710 \n",
"top \n",
"freq 15012 3308 \n",
"mean NaN NaN \n",
"std NaN NaN \n",
"min NaN NaN \n",
"25% NaN NaN \n",
"50% NaN NaN \n",
"75% NaN NaN \n",
"max NaN NaN \n",
added first homework
2022-03-19 23:50:32 +01:00
"\n",
changed Nan to empty strings
2022-03-20 22:09:51 +01:00
" description requirements \\\n",
"count 17880 17880 \n",
"unique 14802 11969 \n",
"top Play with kids, get paid for it Love travel? J... \n",
"freq 379 2695 \n",
"mean NaN NaN \n",
"std NaN NaN \n",
"min NaN NaN \n",
"25% NaN NaN \n",
"50% NaN NaN \n",
"75% NaN NaN \n",
"max NaN NaN \n",
added first homework
2022-03-19 23:50:32 +01:00
"\n",
changed Nan to empty strings
2022-03-20 22:09:51 +01:00
" benefits telecommuting has_company_logo has_questions \\\n",
"count 17880 17880.000000 17880.000000 17880.000000 \n",
"unique 6206 NaN NaN NaN \n",
"top NaN NaN NaN \n",
"freq 7210 NaN NaN NaN \n",
"mean NaN 0.042897 0.795302 0.491723 \n",
"std NaN 0.202631 0.403492 0.499945 \n",
"min NaN 0.000000 0.000000 0.000000 \n",
"25% NaN 0.000000 1.000000 0.000000 \n",
"50% NaN 0.000000 1.000000 0.000000 \n",
"75% NaN 0.000000 1.000000 1.000000 \n",
"max NaN 1.000000 1.000000 1.000000 \n",
added first homework
2022-03-19 23:50:32 +01:00
"\n",
changed Nan to empty strings
2022-03-20 22:09:51 +01:00
" employment_type required_experience required_education industry \\\n",
"count 17880 17880 17880 17880 \n",
"unique 6 8 14 132 \n",
"top Full-time \n",
"freq 11620 7050 8105 4903 \n",
"mean NaN NaN NaN NaN \n",
"std NaN NaN NaN NaN \n",
"min NaN NaN NaN NaN \n",
"25% NaN NaN NaN NaN \n",
"50% NaN NaN NaN NaN \n",
"75% NaN NaN NaN NaN \n",
"max NaN NaN NaN NaN \n",
added first homework
2022-03-19 23:50:32 +01:00
"\n",
changed Nan to empty strings
2022-03-20 22:09:51 +01:00
" function fraudulent \n",
"count 17880 17880.000000 \n",
"unique 38 NaN \n",
"top NaN \n",
"freq 6455 NaN \n",
"mean NaN 0.048434 \n",
"std NaN 0.214688 \n",
"min NaN 0.000000 \n",
"25% NaN 0.000000 \n",
"50% NaN 0.000000 \n",
"75% NaN 0.000000 \n",
"max NaN 1.000000 "
added first homework
2022-03-19 23:50:32 +01:00
]
},
added Jenkinsfile
2022-03-21 11:00:03 +01:00
"execution_count": 37,
added first homework
2022-03-19 23:50:32 +01:00
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"data.describe(include='all')"
]
},
{
"cell_type": "code",
added Jenkinsfile
2022-03-21 11:00:03 +01:00
"execution_count": 38,
refactored
2022-03-19 23:57:55 +01:00
"id": "4b0e77a4",
added first homework
2022-03-19 23:50:32 +01:00
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"job_id 8940.5\n",
"telecommuting 0.0\n",
"has_company_logo 1.0\n",
"has_questions 0.0\n",
"fraudulent 0.0\n",
"dtype: float64"
]
},
added Jenkinsfile
2022-03-21 11:00:03 +01:00
"execution_count": 38,
added first homework
2022-03-19 23:50:32 +01:00
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"data.median()"
]
},
{
"cell_type": "code",
added Jenkinsfile
2022-03-21 11:00:03 +01:00
"execution_count": 39,
refactored
2022-03-19 23:57:55 +01:00
"id": "5a1d8ec7",
added first homework
2022-03-19 23:50:32 +01:00
"metadata": {},
refactoring
2022-03-21 10:06:54 +01:00
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Defaulting to user installation because normal site-packages is not writeable\n",
added Jenkinsfile
2022-03-21 11:00:03 +01:00
"Requirement already satisfied: scikit-learn in /home/students/s444463/.local/lib/python3.8/site-packages (1.0.2)\n",
refactoring
2022-03-21 10:06:54 +01:00
"Requirement already satisfied: numpy>=1.14.6 in /usr/lib/python3/dist-packages (from scikit-learn) (1.17.4)\n",
added Jenkinsfile
2022-03-21 11:00:03 +01:00
"Requirement already satisfied: threadpoolctl>=2.0.0 in /home/students/s444463/.local/lib/python3.8/site-packages (from scikit-learn) (3.1.0)\n",
refactoring
2022-03-21 10:06:54 +01:00
"Requirement already satisfied: joblib>=0.11 in /usr/lib/python3/dist-packages (from scikit-learn) (0.14.0)\n",
added Jenkinsfile
2022-03-21 11:00:03 +01:00
"Requirement already satisfied: scipy>=1.1.0 in /usr/lib/python3/dist-packages (from scikit-learn) (1.3.3)\n",
refactoring
2022-03-21 10:06:54 +01:00
"\u001b[33mWARNING: You are using pip version 21.2.4; however, version 22.0.4 is available.\n",
"You should consider upgrading via the '/usr/bin/python3 -m pip install --upgrade pip' command.\u001b[0m\n",
"Note: you may need to restart the kernel to use updated packages.\n"
]
}
],
added first homework
2022-03-19 23:50:32 +01:00
"source": [
"pip install -U scikit-learn"
]
},
{
"cell_type": "code",
added Jenkinsfile
2022-03-21 11:00:03 +01:00
"execution_count": 40,
refactored
2022-03-19 23:57:55 +01:00
"id": "50813795",
added first homework
2022-03-19 23:50:32 +01:00
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
refactoring
2022-03-21 10:02:31 +01:00
"English Teacher Abroad 230\n",
"Customer Service Associate 106\n",
"Graduates: English Teacher Abroad (Conversational) 96\n",
"English Teacher Abroad 71\n",
"Software Engineer 67\n",
" ... \n",
refactoring
2022-03-21 10:06:54 +01:00
"Jr. Flash & HTML Developer 1\n",
" RNs Needed ASAP - ER & ICU - APPLY NOW!! 1\n",
"Summer interns - 2014/15 1\n",
"Full-Stack Engineer 1\n",
"Contract Product Designer 1\n",
refactoring
2022-03-21 10:02:31 +01:00
"Name: title, Length: 8461, dtype: int64"
added first homework
2022-03-19 23:50:32 +01:00
]
},
added Jenkinsfile
2022-03-21 11:00:03 +01:00
"execution_count": 40,
added first homework
2022-03-19 23:50:32 +01:00
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"from sklearn.model_selection import train_test_split\n",
"import sklearn\n",
refactoring
2022-03-21 10:02:31 +01:00
"data_train, data_test = train_test_split(data, test_size=5000, random_state=1)\n",
"data_dev, data_test = train_test_split(data_test, test_size=2500, random_state=1)\n",
added first homework
2022-03-19 23:50:32 +01:00
"data_train[\"title\"].value_counts()"
]
},
{
"cell_type": "code",
added Jenkinsfile
2022-03-21 11:00:03 +01:00
"execution_count": 41,
refactored
2022-03-19 23:57:55 +01:00
"id": "ea3c9f2e",
added first homework
2022-03-19 23:50:32 +01:00
"metadata": {},
"outputs": [
{
refactoring
2022-03-21 10:02:31 +01:00
"name": "stdout",
"output_type": "stream",
"text": [
"12880\n",
"2500\n",
"2500\n"
]
added first homework
2022-03-19 23:50:32 +01:00
}
],
"source": [
refactoring
2022-03-21 10:02:31 +01:00
"print(len(data_train))\n",
"print(len(data_dev))\n",
"print(len(data_test))"
added first homework
2022-03-19 23:50:32 +01:00
]
},
{
"cell_type": "code",
added Jenkinsfile
2022-03-21 11:00:03 +01:00
"execution_count": 42,
refactored
2022-03-19 23:57:55 +01:00
"id": "b20cc27a",
added first homework
2022-03-19 23:50:32 +01:00
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
refactoring
2022-03-21 10:06:54 +01:00
"English Teacher Abroad 230\n",
"Customer Service Associate 106\n",
"Graduates: English Teacher Abroad (Conversational) 96\n",
"English Teacher Abroad 71\n",
"Software Engineer 67\n",
" ... \n",
"Jr. Flash & HTML Developer 1\n",
" RNs Needed ASAP - ER & ICU - APPLY NOW!! 1\n",
"Summer interns - 2014/15 1\n",
"Full-Stack Engineer 1\n",
"Contract Product Designer 1\n",
"Name: title, Length: 8461, dtype: int64"
added first homework
2022-03-19 23:50:32 +01:00
]
},
added Jenkinsfile
2022-03-21 11:00:03 +01:00
"execution_count": 42,
added first homework
2022-03-19 23:50:32 +01:00
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"data_train[\"title\"].value_counts()"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
wip
2022-03-27 21:43:29 +02:00
"version": "3.8.12"
added first homework
2022-03-19 23:50:32 +01:00
}
},
"nbformat": 4,
"nbformat_minor": 5
}
Reference in New Issue Copy Permalink