{ "cells": [ { "cell_type": "code", "execution_count": null, "id": "5e2107a5", "metadata": {}, "outputs": [], "source": [ "#Skrypt do ściagnięcia zbiory danych\n" ] }, { "cell_type": "code", "execution_count": 3, "id": "bcc889e5", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Requirement already satisfied: kaggle in /home/mikolaj/.local/lib/python3.8/site-packages (1.5.12)\n", "Requirement already satisfied: tqdm in /home/mikolaj/.local/lib/python3.8/site-packages (from kaggle) (4.59.0)\n", "Requirement already satisfied: urllib3 in /usr/lib/python3/dist-packages (from kaggle) (1.22)\n", "Requirement already satisfied: six>=1.10 in /usr/lib/python3/dist-packages (from kaggle) (1.11.0)\n", "Requirement already satisfied: requests in /home/mikolaj/.local/lib/python3.8/site-packages (from kaggle) (2.25.1)\n", "Requirement already satisfied: python-dateutil in /home/mikolaj/.local/lib/python3.8/site-packages (from kaggle) (2.8.1)\n", "Requirement already satisfied: certifi in /usr/lib/python3/dist-packages (from kaggle) (2018.1.18)\n", "Requirement already satisfied: python-slugify in /home/mikolaj/.local/lib/python3.8/site-packages (from kaggle) (6.1.1)\n", "Requirement already satisfied: text-unidecode>=1.3 in /home/mikolaj/.local/lib/python3.8/site-packages (from python-slugify->kaggle) (1.3)\n", "Requirement already satisfied: idna<3,>=2.5 in /home/mikolaj/.local/lib/python3.8/site-packages (from requests->kaggle) (2.10)\n", "Requirement already satisfied: chardet<5,>=3.0.2 in /usr/lib/python3/dist-packages (from requests->kaggle) (3.0.4)\n", "\u001b[33mWARNING: You are using pip version 21.3.1; however, version 22.0.4 is available.\n", "You should consider upgrading via the '/usr/bin/python3 -m pip install --upgrade pip' command.\u001b[0m\n", "Requirement already satisfied: pandas in /home/mikolaj/.local/lib/python3.8/site-packages (1.1.5)\n", "Requirement already satisfied: pytz>=2017.2 in /usr/lib/python3/dist-packages (from pandas) (2018.3)\n", "Requirement already satisfied: numpy>=1.15.4 in /home/mikolaj/.local/lib/python3.8/site-packages (from pandas) (1.19.5)\n", "Requirement already satisfied: python-dateutil>=2.7.3 in /home/mikolaj/.local/lib/python3.8/site-packages (from pandas) (2.8.1)\n", "Requirement already satisfied: six>=1.5 in /usr/lib/python3/dist-packages (from python-dateutil>=2.7.3->pandas) (1.11.0)\n", "\u001b[33mWARNING: You are using pip version 21.3.1; however, version 22.0.4 is available.\n", "You should consider upgrading via the '/usr/bin/python3 -m pip install --upgrade pip' command.\u001b[0m\n" ] } ], "source": [ "!pip install --user kaggle #API Kaggle, do pobrania zbioru\n", "!pip install --user pandas" ] }, { "cell_type": "code", "execution_count": 17, "id": "02a4034f", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Warning: Your Kaggle API key is readable by other users on this system! To fix this, you can run 'chmod 600 /home/mikolaj/.kaggle/kaggle.json'\n", "Downloading real-or-fake-fake-jobposting-prediction.zip to /home/mikolaj/ai_tech/inzynieria\n", " 99%|█████████████████████████████████████▊| 16.0M/16.1M [00:01<00:00, 10.2MB/s]\n", "100%|██████████████████████████████████████| 16.1M/16.1M [00:01<00:00, 9.54MB/s]\n" ] } ], "source": [ "# Żeby poniższa komenda zadziałała, musisz posiadać plik ~/.kaggle/kaggle.json, zawierający Kaggle API token.\n", "# Instrukcje: https://www.kaggle.com/docs/api\n", "!kaggle datasets download -d shivamb/real-or-fake-fake-jobposting-prediction" ] }, { "cell_type": "code", "execution_count": 7, "id": "5035aef0", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Archive: real-or-fake-fake-jobposting-prediction.zip\n", " inflating: fake_job_postings.csv \n" ] } ], "source": [ "!unzip -o real-or-fake-fake-jobposting-prediction.zip" ] }, { "cell_type": "code", "execution_count": 19, "id": "14344d2f", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Collecting seaborn\n", " Downloading seaborn-0.11.2-py3-none-any.whl (292 kB)\n", " |████████████████████████████████| 292 kB 1.8 MB/s \n", "\u001b[?25hRequirement already satisfied: numpy>=1.15 in /home/mikolaj/.local/lib/python3.8/site-packages (from seaborn) (1.19.5)\n", "Requirement already satisfied: scipy>=1.0 in /home/mikolaj/.local/lib/python3.8/site-packages (from seaborn) (1.7.2)\n", "Requirement already satisfied: pandas>=0.23 in /home/mikolaj/.local/lib/python3.8/site-packages (from seaborn) (1.1.5)\n", "Requirement already satisfied: matplotlib>=2.2 in /home/mikolaj/.local/lib/python3.8/site-packages (from seaborn) (3.4.2)\n", "Requirement already satisfied: pyparsing>=2.2.1 in /home/mikolaj/.local/lib/python3.8/site-packages (from matplotlib>=2.2->seaborn) (2.4.7)\n", "Requirement already satisfied: pillow>=6.2.0 in /home/mikolaj/.local/lib/python3.8/site-packages (from matplotlib>=2.2->seaborn) (8.2.0)\n", "Requirement already satisfied: python-dateutil>=2.7 in /home/mikolaj/.local/lib/python3.8/site-packages (from matplotlib>=2.2->seaborn) (2.8.1)\n", "Requirement already satisfied: kiwisolver>=1.0.1 in /home/mikolaj/.local/lib/python3.8/site-packages (from matplotlib>=2.2->seaborn) (1.3.1)\n", "Requirement already satisfied: cycler>=0.10 in /home/mikolaj/.local/lib/python3.8/site-packages (from matplotlib>=2.2->seaborn) (0.10.0)\n", "Requirement already satisfied: pytz>=2017.2 in /usr/lib/python3/dist-packages (from pandas>=0.23->seaborn) (2018.3)\n", "Requirement already satisfied: six in /usr/lib/python3/dist-packages (from cycler>=0.10->matplotlib>=2.2->seaborn) (1.11.0)\n", "Installing collected packages: seaborn\n", "Successfully installed seaborn-0.11.2\n", "\u001b[33mWARNING: You are using pip version 21.3.1; however, version 22.0.4 is available.\n", "You should consider upgrading via the '/usr/bin/python3 -m pip install --upgrade pip' command.\u001b[0m\n" ] } ], "source": [ "!pip install --user seaborn" ] }, { "cell_type": "code", "execution_count": 43, "id": "0f5ebfab", "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
job_idtitlelocationdepartmentsalary_rangecompany_profiledescriptionrequirementsbenefitstelecommutinghas_company_logohas_questionsemployment_typerequired_experiencerequired_educationindustryfunctionfraudulent
01Marketing InternUS, NY, New YorkMarketingNaNWe're Food52, and we've created a groundbreaki...Food52, a fast-growing, James Beard Award-winn...Experience with content management systems a m...NaN010OtherInternshipNaNNaNMarketing0
12Customer Service - Cloud Video ProductionNZ, , AucklandSuccessNaN90 Seconds, the worlds Cloud Video Production ...Organised - Focused - Vibrant - Awesome!Do you...What we expect from you:Your key responsibilit...What you will get from usThrough being part of...010Full-timeNot ApplicableNaNMarketing and AdvertisingCustomer Service0
23Commissioning Machinery Assistant (CMA)US, IA, WeverNaNNaNValor Services provides Workforce Solutions th...Our client, located in Houston, is actively se...Implement pre-commissioning and commissioning ...NaN010NaNNaNNaNNaNNaN0
34Account Executive - Washington DCUS, DC, WashingtonSalesNaNOur passion for improving quality of life thro...THE COMPANY: ESRI – Environmental Systems Rese...EDUCATION: Bachelor’s or Master’s in GIS, busi...Our culture is anything but corporate—we have ...010Full-timeMid-Senior levelBachelor's DegreeComputer SoftwareSales0
45Bill Review ManagerUS, FL, Fort WorthNaNNaNSpotSource Solutions LLC is a Global Human Cap...JOB TITLE: Itemization Review ManagerLOCATION:...QUALIFICATIONS:RN license in the State of Texa...Full Benefits Offered011Full-timeMid-Senior levelBachelor's DegreeHospital & Health CareHealth Care Provider0
.........................................................
1787517876Account Director - DistributionCA, ON, TorontoSalesNaNVend is looking for some awesome new talent to...Just in case this is the first time you’ve vis...To ace this role you:Will eat comprehensive St...What can you expect from us?We have an open cu...011Full-timeMid-Senior levelNaNComputer SoftwareSales0
1787617877Payroll AccountantUS, PA, PhiladelphiaAccountingNaNWebLinc is the e-commerce platform and service...The Payroll Accountant will focus primarily on...- B.A. or B.S. in Accounting- Desire to have f...Health &amp; WellnessMedical planPrescription ...011Full-timeMid-Senior levelBachelor's DegreeInternetAccounting/Auditing0
1787717878Project Cost Control Staff Engineer - Cost Con...US, TX, HoustonNaNNaNWe Provide Full Time Permanent Positions for m...Experienced Project Cost Control Staff Enginee...At least 12 years professional experience.Abil...NaN000Full-timeNaNNaNNaNNaN0
1787817879Graphic DesignerNG, LA, LagosNaNNaNNaNNemsia Studios is looking for an experienced v...1. Must be fluent in the latest versions of Co...Competitive salary (compensation will be based...001ContractNot ApplicableProfessionalGraphic DesignDesign0
1787917880Web Application DevelopersNZ, N, WellingtonEngineeringNaNVend is looking for some awesome new talent to...Who are we?Vend is an award winning web based ...We want to hear from you if:You have an in-dep...NaN011Full-timeMid-Senior levelNaNComputer SoftwareEngineering0
\n", "

17880 rows × 18 columns

\n", "
" ], "text/plain": [ " job_id title \\\n", "0 1 Marketing Intern \n", "1 2 Customer Service - Cloud Video Production \n", "2 3 Commissioning Machinery Assistant (CMA) \n", "3 4 Account Executive - Washington DC \n", "4 5 Bill Review Manager \n", "... ... ... \n", "17875 17876 Account Director - Distribution \n", "17876 17877 Payroll Accountant \n", "17877 17878 Project Cost Control Staff Engineer - Cost Con... \n", "17878 17879 Graphic Designer \n", "17879 17880 Web Application Developers \n", "\n", " location department salary_range \\\n", "0 US, NY, New York Marketing NaN \n", "1 NZ, , Auckland Success NaN \n", "2 US, IA, Wever NaN NaN \n", "3 US, DC, Washington Sales NaN \n", "4 US, FL, Fort Worth NaN NaN \n", "... ... ... ... \n", "17875 CA, ON, Toronto Sales NaN \n", "17876 US, PA, Philadelphia Accounting NaN \n", "17877 US, TX, Houston NaN NaN \n", "17878 NG, LA, Lagos NaN NaN \n", "17879 NZ, N, Wellington Engineering NaN \n", "\n", " company_profile \\\n", "0 We're Food52, and we've created a groundbreaki... \n", "1 90 Seconds, the worlds Cloud Video Production ... \n", "2 Valor Services provides Workforce Solutions th... \n", "3 Our passion for improving quality of life thro... \n", "4 SpotSource Solutions LLC is a Global Human Cap... \n", "... ... \n", "17875 Vend is looking for some awesome new talent to... \n", "17876 WebLinc is the e-commerce platform and service... \n", "17877 We Provide Full Time Permanent Positions for m... \n", "17878 NaN \n", "17879 Vend is looking for some awesome new talent to... \n", "\n", " description \\\n", "0 Food52, a fast-growing, James Beard Award-winn... \n", "1 Organised - Focused - Vibrant - Awesome!Do you... \n", "2 Our client, located in Houston, is actively se... \n", "3 THE COMPANY: ESRI – Environmental Systems Rese... \n", "4 JOB TITLE: Itemization Review ManagerLOCATION:... \n", "... ... \n", "17875 Just in case this is the first time you’ve vis... \n", "17876 The Payroll Accountant will focus primarily on... \n", "17877 Experienced Project Cost Control Staff Enginee... \n", "17878 Nemsia Studios is looking for an experienced v... \n", "17879 Who are we?Vend is an award winning web based ... \n", "\n", " requirements \\\n", "0 Experience with content management systems a m... \n", "1 What we expect from you:Your key responsibilit... \n", "2 Implement pre-commissioning and commissioning ... \n", "3 EDUCATION: Bachelor’s or Master’s in GIS, busi... \n", "4 QUALIFICATIONS:RN license in the State of Texa... \n", "... ... \n", "17875 To ace this role you:Will eat comprehensive St... \n", "17876 - B.A. or B.S. in Accounting- Desire to have f... \n", "17877 At least 12 years professional experience.Abil... \n", "17878 1. Must be fluent in the latest versions of Co... \n", "17879 We want to hear from you if:You have an in-dep... \n", "\n", " benefits telecommuting \\\n", "0 NaN 0 \n", "1 What you will get from usThrough being part of... 0 \n", "2 NaN 0 \n", "3 Our culture is anything but corporate—we have ... 0 \n", "4 Full Benefits Offered 0 \n", "... ... ... \n", "17875 What can you expect from us?We have an open cu... 0 \n", "17876 Health & WellnessMedical planPrescription ... 0 \n", "17877 NaN 0 \n", "17878 Competitive salary (compensation will be based... 0 \n", "17879 NaN 0 \n", "\n", " has_company_logo has_questions employment_type required_experience \\\n", "0 1 0 Other Internship \n", "1 1 0 Full-time Not Applicable \n", "2 1 0 NaN NaN \n", "3 1 0 Full-time Mid-Senior level \n", "4 1 1 Full-time Mid-Senior level \n", "... ... ... ... ... \n", "17875 1 1 Full-time Mid-Senior level \n", "17876 1 1 Full-time Mid-Senior level \n", "17877 0 0 Full-time NaN \n", "17878 0 1 Contract Not Applicable \n", "17879 1 1 Full-time Mid-Senior level \n", "\n", " required_education industry function \\\n", "0 NaN NaN Marketing \n", "1 NaN Marketing and Advertising Customer Service \n", "2 NaN NaN NaN \n", "3 Bachelor's Degree Computer Software Sales \n", "4 Bachelor's Degree Hospital & Health Care Health Care Provider \n", "... ... ... ... \n", "17875 NaN Computer Software Sales \n", "17876 Bachelor's Degree Internet Accounting/Auditing \n", "17877 NaN NaN NaN \n", "17878 Professional Graphic Design Design \n", "17879 NaN Computer Software Engineering \n", "\n", " fraudulent \n", "0 0 \n", "1 0 \n", "2 0 \n", "3 0 \n", "4 0 \n", "... ... \n", "17875 0 \n", "17876 0 \n", "17877 0 \n", "17878 0 \n", "17879 0 \n", "\n", "[17880 rows x 18 columns]" ] }, "execution_count": 43, "metadata": {}, "output_type": "execute_result" } ], "source": [ "import pandas as pd\n", "data=pd.read_csv('fake_job_postings.csv')\n", "data\n" ] }, { "cell_type": "code", "execution_count": 20, "id": "f2605822", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "61327 real-or-fake-fake-jobposting-prediction.zip\r\n" ] } ], "source": [ "#Wielkosc zbioru\n", "!wc -l real-or-fake-fake-jobposting-prediction.zip" ] }, { "cell_type": "code", "execution_count": 35, "id": "a805a6f4", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "English Teacher Abroad 311\n", "Customer Service Associate 146\n", "Graduates: English Teacher Abroad (Conversational) 144\n", "English Teacher Abroad 95\n", "Software Engineer 86\n", " ... \n", "Welcome desk Administrator 1\n", "Game Studio Manager 1\n", "Head of Content (m/f) 1\n", "International Broadcaster, Bambara and Songhai Language 1\n", "Community / Marketing Coordinator 1\n", "Name: title, Length: 11231, dtype: int64" ] }, "execution_count": 35, "metadata": {}, "output_type": "execute_result" } ], "source": [ "data[\"title\"].value_counts()" ] }, { "cell_type": "code", "execution_count": 36, "id": "2aee75f0", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "Sales 551\n", "Engineering 487\n", "Marketing 401\n", "Operations 270\n", "IT 225\n", " ... \n", "Audiology 1\n", "Professional Association 1\n", "Multiwork 1\n", "Dev2 to 5 years experience in a client-facing technical role. Ability to diplomatically address customer concerns and provide feedback. A firm understanding of the technology stacks common to the Web ecosystem. A demonstrated history of creating non-trivi 1\n", "Business Strategy 1\n", "Name: department, Length: 1337, dtype: int64" ] }, "execution_count": 36, "metadata": {}, "output_type": "execute_result" } ], "source": [ "data[\"department\"].value_counts()" ] }, { "cell_type": "code", "execution_count": 37, "id": "06e51205", "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
job_idtitlelocationdepartmentsalary_rangecompany_profiledescriptionrequirementsbenefitstelecommutinghas_company_logohas_questionsemployment_typerequired_experiencerequired_educationindustryfunctionfraudulent
count17880.0000001788017534633328681457217879151851067017880.00000017880.00000017880.00000014409108309775129771142517880.000000
uniqueNaN1123131051337874170914801119686205NaNNaNNaN571313137NaN
topNaNEnglish Teacher AbroadGB, LND, LondonSales0-0We help teachers get safe &amp; secure jobs ab...Play with kids, get paid for it Love travel? J...University degree required. TEFL / TESOL / CEL...See job descriptionNaNNaNNaNFull-timeMid-Senior levelBachelor's DegreeInformation Technology and ServicesInformation TechnologyNaN
freqNaN311718551142726379410726NaNNaNNaN116203809514517341749NaN
mean8940.500000NaNNaNNaNNaNNaNNaNNaNNaN0.0428970.7953020.491723NaNNaNNaNNaNNaN0.048434
std5161.655742NaNNaNNaNNaNNaNNaNNaNNaN0.2026310.4034920.499945NaNNaNNaNNaNNaN0.214688
min1.000000NaNNaNNaNNaNNaNNaNNaNNaN0.0000000.0000000.000000NaNNaNNaNNaNNaN0.000000
25%4470.750000NaNNaNNaNNaNNaNNaNNaNNaN0.0000001.0000000.000000NaNNaNNaNNaNNaN0.000000
50%8940.500000NaNNaNNaNNaNNaNNaNNaNNaN0.0000001.0000000.000000NaNNaNNaNNaNNaN0.000000
75%13410.250000NaNNaNNaNNaNNaNNaNNaNNaN0.0000001.0000001.000000NaNNaNNaNNaNNaN0.000000
max17880.000000NaNNaNNaNNaNNaNNaNNaNNaN1.0000001.0000001.000000NaNNaNNaNNaNNaN1.000000
\n", "
" ], "text/plain": [ " job_id title location department \\\n", "count 17880.000000 17880 17534 6333 \n", "unique NaN 11231 3105 1337 \n", "top NaN English Teacher Abroad GB, LND, London Sales \n", "freq NaN 311 718 551 \n", "mean 8940.500000 NaN NaN NaN \n", "std 5161.655742 NaN NaN NaN \n", "min 1.000000 NaN NaN NaN \n", "25% 4470.750000 NaN NaN NaN \n", "50% 8940.500000 NaN NaN NaN \n", "75% 13410.250000 NaN NaN NaN \n", "max 17880.000000 NaN NaN NaN \n", "\n", " salary_range company_profile \\\n", "count 2868 14572 \n", "unique 874 1709 \n", "top 0-0 We help teachers get safe & secure jobs ab... \n", "freq 142 726 \n", "mean NaN NaN \n", "std NaN NaN \n", "min NaN NaN \n", "25% NaN NaN \n", "50% NaN NaN \n", "75% NaN NaN \n", "max NaN NaN \n", "\n", " description \\\n", "count 17879 \n", "unique 14801 \n", "top Play with kids, get paid for it Love travel? J... \n", "freq 379 \n", "mean NaN \n", "std NaN \n", "min NaN \n", "25% NaN \n", "50% NaN \n", "75% NaN \n", "max NaN \n", "\n", " requirements \\\n", "count 15185 \n", "unique 11968 \n", "top University degree required. TEFL / TESOL / CEL... \n", "freq 410 \n", "mean NaN \n", "std NaN \n", "min NaN \n", "25% NaN \n", "50% NaN \n", "75% NaN \n", "max NaN \n", "\n", " benefits telecommuting has_company_logo has_questions \\\n", "count 10670 17880.000000 17880.000000 17880.000000 \n", "unique 6205 NaN NaN NaN \n", "top See job description NaN NaN NaN \n", "freq 726 NaN NaN NaN \n", "mean NaN 0.042897 0.795302 0.491723 \n", "std NaN 0.202631 0.403492 0.499945 \n", "min NaN 0.000000 0.000000 0.000000 \n", "25% NaN 0.000000 1.000000 0.000000 \n", "50% NaN 0.000000 1.000000 0.000000 \n", "75% NaN 0.000000 1.000000 1.000000 \n", "max NaN 1.000000 1.000000 1.000000 \n", "\n", " employment_type required_experience required_education \\\n", "count 14409 10830 9775 \n", "unique 5 7 13 \n", "top Full-time Mid-Senior level Bachelor's Degree \n", "freq 11620 3809 5145 \n", "mean NaN NaN NaN \n", "std NaN NaN NaN \n", "min NaN NaN NaN \n", "25% NaN NaN NaN \n", "50% NaN NaN NaN \n", "75% NaN NaN NaN \n", "max NaN NaN NaN \n", "\n", " industry function \\\n", "count 12977 11425 \n", "unique 131 37 \n", "top Information Technology and Services Information Technology \n", "freq 1734 1749 \n", "mean NaN NaN \n", "std NaN NaN \n", "min NaN NaN \n", "25% NaN NaN \n", "50% NaN NaN \n", "75% NaN NaN \n", "max NaN NaN \n", "\n", " fraudulent \n", "count 17880.000000 \n", "unique NaN \n", "top NaN \n", "freq NaN \n", "mean 0.048434 \n", "std 0.214688 \n", "min 0.000000 \n", "25% 0.000000 \n", "50% 0.000000 \n", "75% 0.000000 \n", "max 1.000000 " ] }, "execution_count": 37, "metadata": {}, "output_type": "execute_result" } ], "source": [ "data.describe(include='all')" ] }, { "cell_type": "code", "execution_count": 39, "id": "cd4321ee", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "job_id 8940.5\n", "telecommuting 0.0\n", "has_company_logo 1.0\n", "has_questions 0.0\n", "fraudulent 0.0\n", "dtype: float64" ] }, "execution_count": 39, "metadata": {}, "output_type": "execute_result" } ], "source": [ "data.median()" ] }, { "cell_type": "code", "execution_count": 19, "id": "74619509", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Defaulting to user installation because normal site-packages is not writeable\n", "Collecting scikit-learn\n", " Downloading scikit_learn-1.0.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (26.7 MB)\n", " |████████████████████████████████| 26.7 MB 8.8 MB/s \n", "\u001b[?25hRequirement already satisfied: numpy>=1.14.6 in /home/mikolaj/.local/lib/python3.8/site-packages (from scikit-learn) (1.19.5)\n", "Requirement already satisfied: scipy>=1.1.0 in /home/mikolaj/.local/lib/python3.8/site-packages (from scikit-learn) (1.7.2)\n", "Collecting threadpoolctl>=2.0.0\n", " Downloading threadpoolctl-3.1.0-py3-none-any.whl (14 kB)\n", "Requirement already satisfied: joblib>=0.11 in /home/mikolaj/.local/lib/python3.8/site-packages (from scikit-learn) (1.1.0)\n", "Installing collected packages: threadpoolctl, scikit-learn\n", "Successfully installed scikit-learn-1.0.2 threadpoolctl-3.1.0\n", "\u001b[33mWARNING: You are using pip version 21.3.1; however, version 22.0.4 is available.\n", "You should consider upgrading via the '/usr/bin/python3 -m pip install --upgrade pip' command.\u001b[0m\n", "Note: you may need to restart the kernel to use updated packages.\n" ] } ], "source": [ "pip install -U scikit-learn" ] }, { "cell_type": "code", "execution_count": 26, "id": "17c852ed", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "English Teacher Abroad 235\n", "Customer Service Associate 110\n", "Graduates: English Teacher Abroad (Conversational) 104\n", "English Teacher Abroad 72\n", "Software Engineer 68\n", " ... \n", "Manager-Plastics Mfg Engineering - Full Time Permanent Job 1\n", "Ruby on Rails Developer/Programmer 1\n", "Appliance Technician 1\n", "Need Oracle Fusion HCM Resource 1\n", "Recruitment specialist 1\n", "Name: title, Length: 8761, dtype: int64" ] }, "execution_count": 26, "metadata": {}, "output_type": "execute_result" } ], "source": [ "from sklearn.model_selection import train_test_split\n", "import sklearn\n", "data_train, data_test = train_test_split(data, random_state=1)\n", "data_train[\"title\"].value_counts()" ] }, { "cell_type": "code", "execution_count": 31, "id": "618a6004", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "0.3333333333333333" ] }, "execution_count": 31, "metadata": {}, "output_type": "execute_result" } ], "source": [ "data_test.size/data_train.size" ] }, { "cell_type": "code", "execution_count": 32, "id": "1a182b53", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "English Teacher Abroad 235\n", "Customer Service Associate 110\n", "Graduates: English Teacher Abroad (Conversational) 104\n", "English Teacher Abroad 72\n", "Software Engineer 68\n", " ... \n", "Manager-Plastics Mfg Engineering - Full Time Permanent Job 1\n", "Ruby on Rails Developer/Programmer 1\n", "Appliance Technician 1\n", "Need Oracle Fusion HCM Resource 1\n", "Recruitment specialist 1\n", "Name: title, Length: 8761, dtype: int64" ] }, "execution_count": 32, "metadata": {}, "output_type": "execute_result" } ], "source": [ "data_train[\"title\"].value_counts()" ] } ], "metadata": { "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.8.12" } }, "nbformat": 4, "nbformat_minor": 5 }