diff --git a/Car4You_webscrapper/pom.xml b/Car4You_webscrapper/pom.xml new file mode 100644 index 0000000..3406f77 --- /dev/null +++ b/Car4You_webscrapper/pom.xml @@ -0,0 +1,27 @@ + + + 4.0.0 + + groupId + Car4You_webscrapper + 1.0-SNAPSHOT + + + + + org.jsoup + jsoup + 1.12.1 + + + + + postgresql + postgresql + 9.1-901-1.jdbc4 + + + + \ No newline at end of file diff --git a/Car4You_webscrapper/src/main/java/CrawlerThread.java b/Car4You_webscrapper/src/main/java/CrawlerThread.java new file mode 100644 index 0000000..40aa692 --- /dev/null +++ b/Car4You_webscrapper/src/main/java/CrawlerThread.java @@ -0,0 +1,122 @@ +import org.jsoup.Jsoup; +import org.jsoup.nodes.Document; +import org.jsoup.nodes.Element; +import org.jsoup.select.Elements; + +import java.io.IOException; +import java.util.Hashtable; + +public class CrawlerThread extends Thread { + + public CrawlerThread(){ + + } + + @Override + public void run() { + + String pageUrl = "https://www.otomoto.pl/osobowe/?search%5Bfilter_enum_has_registration%5D=1&search%5Bfilter_enum_damaged%5D=0&search%5Bfilter_enum_registered%5D=1&search%5Bfilter_enum_no_accident%5D=1&search%5Border%5D=filter_float_price%3Aasc&search%5Bbrand_program_id%5D%5B0%5D=&search%5Bcountry%5D=&page="; + for (int i = 1; i <= 500; i++) { + Document doc = null; + System.out.println("Strona nr: " + i); + String targetUrl = pageUrl + Integer.valueOf(i); + System.out.println("Link: " + targetUrl); + Boolean success = false; //flaga do sprawdzania czy udało sie połączyc + while (!success) { + success = true; + try { + doc = Jsoup.connect(targetUrl).get(); + } catch (IOException e) { + System.out.println("Nie udalo sie polaczyc. " + e.getMessage()); + success = false; + } + } + + Elements links = doc.select("a.offer-title__link"); + + for (Element element : links) { + Element photoDiv = element.parent().parent().parent().parent().selectFirst("div").selectFirst("span"); +// System.out.println("Promowana: " + photoDiv); + if (photoDiv == null) { //todo sprawdzać czy oferta jest promowana + String url = element.attr("href"); + System.out.println("Pobieram: " + url); + Document pageDoc = null; + success = false; //flaga do sprawdzania czy udało sie połączyc + while (!success) { + success = true; + try { + pageDoc = Jsoup.connect(url).get(); + } catch (IOException e) { + System.out.println("Nie udalo sie polaczyc. " + e.getMessage()); + success = false; + } + } + Elements offerParamsList = pageDoc.select("ul.offer-params__list").select("li.offer-params__item"); + Hashtable paramsHashtable = new Hashtable(); + + String priceString = pageDoc.select("span.offer-price__number").first().text(); + if(!priceString.contains(",")) { + priceString = priceString.substring(0, priceString.length() - 4); + priceString = priceString.replaceAll(" ", ""); + }else{ + priceString = priceString.replaceAll(",", ""); + priceString = priceString.substring(0, priceString.length()-2); + priceString = priceString.substring(0, priceString.length() - 4); + priceString = priceString.replaceAll(" ", ""); + } + Integer price = Integer.valueOf(priceString); + + for (Element param : offerParamsList) { //wyciąganie wartości parametrów do tablicy + String key = param.select("span.offer-params__label").text(); + String value = param.select("div.offer-params__value").text(); + paramsHashtable.put(key, value); + } + + try { + String brand = paramsHashtable.get("Marka pojazdu"); + String model = paramsHashtable.get("Model pojazdu"); + String version = paramsHashtable.get("Wersja"); + Integer year = Integer.valueOf(paramsHashtable.get("Rok produkcji")); + Integer enginePower = Integer.valueOf(paramsHashtable.get("Moc").split(" ")[0]); + String engineCapacityString = paramsHashtable.get("Pojemność skokowa").replaceAll(" ", ""); + engineCapacityString = engineCapacityString.substring(0, engineCapacityString.length() - 3); + Double engineCapacity = Double.valueOf(engineCapacityString); + String fuel = paramsHashtable.get("Rodzaj paliwa"); + String transmission = paramsHashtable.get("Skrzynia biegów"); + String drive = paramsHashtable.get("Napęd"); + String bodyType = paramsHashtable.get("Typ"); + Integer doors = Integer.valueOf(paramsHashtable.get("Liczba drzwi")); + Integer seats = Integer.valueOf(paramsHashtable.get("Liczba miejsc")); + + if(year == null) year = 0; + if(version == null) version = "nieokreślono"; + if(drive == null) drive = "nieokreślono"; + if(transmission == null) transmission = "nieokreślono"; + +// System.out.println("Samochód: " + +// "\nmarka: " + brand + +// "\nmodel: " + model + +// "\nwersja: " + version + +// "\nrok produkcji: " + year + +// "\nmoc silnika: " + enginePower + +// "\npojemnosc skokoa: " + engineCapacity + +// "\nrodzaj paliwa; " + fuel + +// "\nskrzynia biegow: " + transmission + +// "\nnapęd: " + drive + +// "\ntyp nadwozia: " + bodyType + +// "\nliczba drzwi: " + doors + +// "\nliczba miejsc: " + seats + +// "\ncena: " + price); + + dbService.insert(brand, model, version, year, enginePower, engineCapacity, fuel, transmission, drive, bodyType, doors, seats, price); + } catch (Exception e) { + System.out.println("Błąd pobierania danych!: " + e.getMessage()); + } +// break; + }else{ + System.out.println("Oferta promowana. Pomijam. "+element.text()); + } + } + } + } +} diff --git a/Car4You_webscrapper/src/main/java/ThreadManager.java b/Car4You_webscrapper/src/main/java/ThreadManager.java new file mode 100644 index 0000000..1032716 --- /dev/null +++ b/Car4You_webscrapper/src/main/java/ThreadManager.java @@ -0,0 +1,21 @@ +import java.util.ArrayList; + +public class ThreadManager { + + public static void main(String[] args) { + + ArrayList threads = new ArrayList(); + + //tworzenie watkow + for(int i = 0; i < 1; i++){ + threads.add(new CrawlerThread()); + } + + //uruchomienie watkow + for(CrawlerThread t:threads){ + t.start(); + } + + } + +} diff --git a/Car4You_webscrapper/src/main/java/dbService.java b/Car4You_webscrapper/src/main/java/dbService.java new file mode 100644 index 0000000..448732d --- /dev/null +++ b/Car4You_webscrapper/src/main/java/dbService.java @@ -0,0 +1,115 @@ +import java.sql.*; + +public class dbService { + // JDBC driver name and database URL + static final String JDBC_DRIVER = "com.postgresql.jdbc.Driver"; + static final String DB_URL = ""; + + // Database credentials + static final String USER = ""; + static final String PASS = ""; + + public static void insert(String brand, String model, String version, Integer year, Integer enginePower, Double engineCapacity, String fuel, String transmission, String drive, String bodyType, Integer doors, Integer seats, Integer price) { + Connection conn = null; + try{ + conn = DriverManager.getConnection(DB_URL,USER,PASS); + + //najpierw sprwadzac czy takie auto juz jest + String selectString = "" + + "SELECT * " + + "FROM car " + + "WHERE " + + "brand = ? AND " + + "model = ? AND " + + "version = ? AND " + + "engine_power = ? AND " + + "engine_capacity = ? AND " + + "fuel = ? AND " + + "transmission = ? AND " + + "drive = ? AND " + + "body_type = ? AND " + + "doors = ? AND " + + "seats = ?"; + PreparedStatement preparedSelect = conn.prepareStatement(selectString); + preparedSelect.setString(1, brand); + preparedSelect.setString(2, model); + preparedSelect.setString(3, version); + preparedSelect.setInt(4, enginePower); + preparedSelect.setDouble(5, engineCapacity); + preparedSelect.setString(6, fuel); + preparedSelect.setString(7, transmission); + preparedSelect.setString(8, drive); + preparedSelect.setString(9, bodyType); + preparedSelect.setInt(10, doors); + preparedSelect.setInt(11, seats); + + ResultSet rs = preparedSelect.executeQuery(); + if(rs.next()) { + try { + Long repeatedId = rs.getLong("id"); + Integer minPrice = rs.getInt("price_from"); + Integer maxPrice = rs.getInt("price_to"); + Long priceSum = rs.getLong("price_sum"); + Long counter = rs.getLong("counter"); + Double avarage; + if (price >= maxPrice) { + maxPrice = price; + } else { + minPrice = price; + } + priceSum = priceSum + price; + counter = counter + 1; + avarage = priceSum / (double) counter; + + //update z nowymi danymi / zmiana tylko 5 parametrow + String updateString = "" + + "UPDATE car " + + "SET price_from = ?, price_to = ?, price_sum = ?, avarage = ?, counter = ? WHERE id = ?"; + PreparedStatement updatePrepare = conn.prepareStatement(updateString); + updatePrepare.setInt(1, minPrice); + updatePrepare.setInt(2, maxPrice); + updatePrepare.setLong(3, priceSum); + updatePrepare.setDouble(4, avarage); + updatePrepare.setLong(5, counter); + updatePrepare.setLong(6, repeatedId); + + int rows = updatePrepare.executeUpdate(); + //zaktualizowano + System.out.println("Zauktualizowano: " + brand); + }catch (Exception e){ + System.out.println("Błąd podczas aktualizacji ogłoszenia: " + e.getMessage()); + } + }else{ + String sql = "INSERT INTO car (brand, model, version, year, engine_power, engine_capacity, fuel, transmission, drive, body_type, doors, seats, price_from, price_to, price_sum, counter, avarage) " + + "values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"; + + PreparedStatement preparedStatement = conn.prepareStatement(sql); + preparedStatement.setString(1, brand); + preparedStatement.setString(2, model); + preparedStatement.setString(3, version); + preparedStatement.setInt(4, year); + preparedStatement.setInt(5, enginePower); + preparedStatement.setDouble(6, engineCapacity); + preparedStatement.setString(7, fuel); + preparedStatement.setString(8, transmission); + preparedStatement.setString(9, drive); + preparedStatement.setString(10, bodyType); + preparedStatement.setInt(11, doors); + preparedStatement.setInt(12, seats); + preparedStatement.setInt(13, price); + preparedStatement.setInt(14, price); + preparedStatement.setInt(15, price); + preparedStatement.setInt(16, 1); + preparedStatement.setDouble(17, price); + + int row = preparedStatement.executeUpdate(); + System.out.println("Dodano: " + brand); + + preparedStatement.close(); + conn.close(); + } + }catch(SQLException se){ + se.printStackTrace(); + } + } +} diff --git a/cars4you/src/main/client/css/animate.min.css b/cars4you/src/main/client/html/css/animate.min.css similarity index 100% rename from cars4you/src/main/client/css/animate.min.css rename to cars4you/src/main/client/html/css/animate.min.css diff --git a/cars4you/src/main/client/css/bootstrap.css b/cars4you/src/main/client/html/css/bootstrap.css similarity index 100% rename from cars4you/src/main/client/css/bootstrap.css rename to cars4you/src/main/client/html/css/bootstrap.css diff --git a/cars4you/src/main/client/css/bootstrap/bootstrap-grid.css b/cars4you/src/main/client/html/css/bootstrap/bootstrap-grid.css similarity index 100% rename from cars4you/src/main/client/css/bootstrap/bootstrap-grid.css rename to cars4you/src/main/client/html/css/bootstrap/bootstrap-grid.css diff --git a/cars4you/src/main/client/css/bootstrap/bootstrap-reboot.css b/cars4you/src/main/client/html/css/bootstrap/bootstrap-reboot.css similarity index 100% rename from cars4you/src/main/client/css/bootstrap/bootstrap-reboot.css rename to cars4you/src/main/client/html/css/bootstrap/bootstrap-reboot.css diff --git a/cars4you/src/main/client/css/bootstrap/bootstrap.css b/cars4you/src/main/client/html/css/bootstrap/bootstrap.css similarity index 100% rename from cars4you/src/main/client/css/bootstrap/bootstrap.css rename to cars4you/src/main/client/html/css/bootstrap/bootstrap.css diff --git a/cars4you/src/main/client/css/font-awesome.min.css b/cars4you/src/main/client/html/css/font-awesome.min.css similarity index 100% rename from cars4you/src/main/client/css/font-awesome.min.css rename to cars4you/src/main/client/html/css/font-awesome.min.css diff --git a/cars4you/src/main/client/css/jquery-ui.css b/cars4you/src/main/client/html/css/jquery-ui.css similarity index 100% rename from cars4you/src/main/client/css/jquery-ui.css rename to cars4you/src/main/client/html/css/jquery-ui.css diff --git a/cars4you/src/main/client/css/linearicons.css b/cars4you/src/main/client/html/css/linearicons.css similarity index 100% rename from cars4you/src/main/client/css/linearicons.css rename to cars4you/src/main/client/html/css/linearicons.css diff --git a/cars4you/src/main/client/css/magnific-popup.css b/cars4you/src/main/client/html/css/magnific-popup.css similarity index 100% rename from cars4you/src/main/client/css/magnific-popup.css rename to cars4you/src/main/client/html/css/magnific-popup.css diff --git a/cars4you/src/main/client/css/main.css b/cars4you/src/main/client/html/css/main.css similarity index 100% rename from cars4you/src/main/client/css/main.css rename to cars4you/src/main/client/html/css/main.css diff --git a/cars4you/src/main/client/css/nice-select.css b/cars4you/src/main/client/html/css/nice-select.css similarity index 100% rename from cars4you/src/main/client/css/nice-select.css rename to cars4you/src/main/client/html/css/nice-select.css diff --git a/cars4you/src/main/client/fonts/FontAwesome.otf b/cars4you/src/main/client/html/fonts/FontAwesome.otf similarity index 100% rename from cars4you/src/main/client/fonts/FontAwesome.otf rename to cars4you/src/main/client/html/fonts/FontAwesome.otf diff --git a/cars4you/src/main/client/fonts/Linearicons-Free.eot b/cars4you/src/main/client/html/fonts/Linearicons-Free.eot similarity index 100% rename from cars4you/src/main/client/fonts/Linearicons-Free.eot rename to cars4you/src/main/client/html/fonts/Linearicons-Free.eot diff --git a/cars4you/src/main/client/fonts/Linearicons-Free.svg b/cars4you/src/main/client/html/fonts/Linearicons-Free.svg similarity index 100% rename from cars4you/src/main/client/fonts/Linearicons-Free.svg rename to cars4you/src/main/client/html/fonts/Linearicons-Free.svg diff --git a/cars4you/src/main/client/fonts/Linearicons-Free.ttf b/cars4you/src/main/client/html/fonts/Linearicons-Free.ttf similarity index 100% rename from cars4you/src/main/client/fonts/Linearicons-Free.ttf rename to cars4you/src/main/client/html/fonts/Linearicons-Free.ttf diff --git a/cars4you/src/main/client/fonts/Linearicons-Free.woff b/cars4you/src/main/client/html/fonts/Linearicons-Free.woff similarity index 100% rename from cars4you/src/main/client/fonts/Linearicons-Free.woff rename to cars4you/src/main/client/html/fonts/Linearicons-Free.woff diff --git a/cars4you/src/main/client/fonts/Linearicons-Free.woff2 b/cars4you/src/main/client/html/fonts/Linearicons-Free.woff2 similarity index 100% rename from cars4you/src/main/client/fonts/Linearicons-Free.woff2 rename to cars4you/src/main/client/html/fonts/Linearicons-Free.woff2 diff --git a/cars4you/src/main/client/fonts/fontawesome-webfont.eot b/cars4you/src/main/client/html/fonts/fontawesome-webfont.eot similarity index 100% rename from cars4you/src/main/client/fonts/fontawesome-webfont.eot rename to cars4you/src/main/client/html/fonts/fontawesome-webfont.eot diff --git a/cars4you/src/main/client/fonts/fontawesome-webfont.svg b/cars4you/src/main/client/html/fonts/fontawesome-webfont.svg similarity index 100% rename from cars4you/src/main/client/fonts/fontawesome-webfont.svg rename to cars4you/src/main/client/html/fonts/fontawesome-webfont.svg diff --git a/cars4you/src/main/client/fonts/fontawesome-webfont.ttf b/cars4you/src/main/client/html/fonts/fontawesome-webfont.ttf similarity index 100% rename from cars4you/src/main/client/fonts/fontawesome-webfont.ttf rename to cars4you/src/main/client/html/fonts/fontawesome-webfont.ttf diff --git a/cars4you/src/main/client/fonts/fontawesome-webfont.woff b/cars4you/src/main/client/html/fonts/fontawesome-webfont.woff similarity index 100% rename from cars4you/src/main/client/fonts/fontawesome-webfont.woff rename to cars4you/src/main/client/html/fonts/fontawesome-webfont.woff diff --git a/cars4you/src/main/client/fonts/fontawesome-webfont.woff2 b/cars4you/src/main/client/html/fonts/fontawesome-webfont.woff2 similarity index 100% rename from cars4you/src/main/client/fonts/fontawesome-webfont.woff2 rename to cars4you/src/main/client/html/fonts/fontawesome-webfont.woff2 diff --git a/cars4you/src/main/client/img/car.jpg b/cars4you/src/main/client/html/img/car.jpg similarity index 100% rename from cars4you/src/main/client/img/car.jpg rename to cars4you/src/main/client/html/img/car.jpg diff --git a/cars4you/src/main/client/img/car4you.png b/cars4you/src/main/client/html/img/car4you.png similarity index 100% rename from cars4you/src/main/client/img/car4you.png rename to cars4you/src/main/client/html/img/car4you.png diff --git a/cars4you/src/main/client/img/header-bg.jpg b/cars4you/src/main/client/html/img/header-bg.jpg similarity index 100% rename from cars4you/src/main/client/img/header-bg.jpg rename to cars4you/src/main/client/html/img/header-bg.jpg diff --git a/cars4you/src/main/client/html/index.html b/cars4you/src/main/client/html/index.html index c569563..6b1a1ef 100644 --- a/cars4you/src/main/client/html/index.html +++ b/cars4you/src/main/client/html/index.html @@ -38,7 +38,7 @@ @@ -63,7 +63,7 @@

W naszej wyszukiwarce go znajdziesz!

- Rozpocznij + Rozpocznij diff --git a/cars4you/src/main/client/html/js/sendingRequest.js b/cars4you/src/main/client/html/js/sendingRequest.js new file mode 100644 index 0000000..e9f08f1 --- /dev/null +++ b/cars4you/src/main/client/html/js/sendingRequest.js @@ -0,0 +1,78 @@ +function sendJSON() { + let result = document.querySelector('.result'); + let fuel_type = document.getElementById('fuel_type'); + let seats = document.getElementsByClassName('seats')[0]; + let doors = document.getElementById('doors'); + let maximum_price = document.getElementById('maximum_price'); + let max_years_old = document.getElementById('max_years_old'); + let engine_power_from = document.getElementById('engine_power_from'); + let engine_power_to = document.getElementById('engine_power_to'); + let body_type = document.getElementById('body_type'); + let gearbox = document.getElementById('gearbox'); + let drive_type = document.getElementById('drive_type'); + + + let req = new XMLHttpRequest(); + // let url = "https://jsonplaceholder.typicode.com/photos"; + // let url = "https://webhook.site/b1792c3d-cd5a-4c29-8e9c-1ff9f84653eb"; + let url = "http://34.65.132.148:8080"; + var params = []; + + if(fuel_type.value != -1){ + params.push("fuel_type="+fuel_type.options[fuel_type.value].text); + } + if(seats.value > 0){ + params.push("seats="+seats.value); + } + if(doors.value != -1){ + params.push("doors="+doors.options[doors.value].text); + } + if(maximum_price.value > 0){ + params.push("maximum_price="+maximum_price.value); + } + if(max_years_old.value > 0){ + params.push("max_years_old="+max_years_old.value); + } + if(engine_power_from.value > 0){ + params.push("engine_power_from="+engine_power_from.value); + } + if(engine_power_to.value > 0){ + params.push("engine_power_to="+engine_power_to.value); + } + if(body_type.value != -1){ + params.push("body_type="+body_type.value); + } + if(gearbox.value != -1){ + params.push("gearbox="+gearbox.value); + } + if(drive_type.value != -1){ + params.push("drive_type="+drive_type.value); + } + + var buff = ""; + if(params.length>0){ + buff += "?"; + for (var i = 0, len = params.length; i < len; i++) { + buff += params[i]; + buff += "\&"; + } + buff = buff.substr(0, buff.length-1); //obcinanie ostatego znaku + } + + console.log("Debug: " + buff); + + // Create a state change callback + req.open("GET", url+buff, true); + req.setRequestHeader("Content-Type", "text/html"); + req.send(); + + req.onreadystatechange = function () { + if (req.readyState === 4 && req.status === 200) { + + // Print received data from server + result.innerHTML = this.responseText; + } + }; + +} + diff --git a/cars4you/src/main/client/lib/bootstrap.min.js b/cars4you/src/main/client/html/lib/bootstrap.min.js similarity index 100% rename from cars4you/src/main/client/lib/bootstrap.min.js rename to cars4you/src/main/client/html/lib/bootstrap.min.js diff --git a/cars4you/src/main/client/lib/easing.min.js b/cars4you/src/main/client/html/lib/easing.min.js similarity index 100% rename from cars4you/src/main/client/lib/easing.min.js rename to cars4you/src/main/client/html/lib/easing.min.js diff --git a/cars4you/src/main/client/lib/hoverIntent.js b/cars4you/src/main/client/html/lib/hoverIntent.js similarity index 100% rename from cars4you/src/main/client/lib/hoverIntent.js rename to cars4you/src/main/client/html/lib/hoverIntent.js diff --git a/cars4you/src/main/client/lib/jquery-2.2.4.min.js b/cars4you/src/main/client/html/lib/jquery-2.2.4.min.js similarity index 100% rename from cars4you/src/main/client/lib/jquery-2.2.4.min.js rename to cars4you/src/main/client/html/lib/jquery-2.2.4.min.js diff --git a/cars4you/src/main/client/lib/jquery.ajaxchimp.min.js b/cars4you/src/main/client/html/lib/jquery.ajaxchimp.min.js similarity index 100% rename from cars4you/src/main/client/lib/jquery.ajaxchimp.min.js rename to cars4you/src/main/client/html/lib/jquery.ajaxchimp.min.js diff --git a/cars4you/src/main/client/lib/jquery.magnific-popup.min.js b/cars4you/src/main/client/html/lib/jquery.magnific-popup.min.js similarity index 100% rename from cars4you/src/main/client/lib/jquery.magnific-popup.min.js rename to cars4you/src/main/client/html/lib/jquery.magnific-popup.min.js diff --git a/cars4you/src/main/client/lib/jquery.nice-select.min.js b/cars4you/src/main/client/html/lib/jquery.nice-select.min.js similarity index 100% rename from cars4you/src/main/client/lib/jquery.nice-select.min.js rename to cars4you/src/main/client/html/lib/jquery.nice-select.min.js diff --git a/cars4you/src/main/client/lib/mail-script.js b/cars4you/src/main/client/html/lib/mail-script.js similarity index 100% rename from cars4you/src/main/client/lib/mail-script.js rename to cars4you/src/main/client/html/lib/mail-script.js diff --git a/cars4you/src/main/client/lib/main.js b/cars4you/src/main/client/html/lib/main.js similarity index 100% rename from cars4you/src/main/client/lib/main.js rename to cars4you/src/main/client/html/lib/main.js diff --git a/cars4you/src/main/client/lib/superfish.min.js b/cars4you/src/main/client/html/lib/superfish.min.js similarity index 100% rename from cars4you/src/main/client/lib/superfish.min.js rename to cars4you/src/main/client/html/lib/superfish.min.js diff --git a/cars4you/src/main/client/html/ankieta.html b/cars4you/src/main/client/html/search.html similarity index 50% rename from cars4you/src/main/client/html/ankieta.html rename to cars4you/src/main/client/html/search.html index e2300e1..b40d45d 100644 --- a/cars4you/src/main/client/html/ankieta.html +++ b/cars4you/src/main/client/html/search.html @@ -31,15 +31,14 @@