임의 디텍토리 생성

$npm init -y

모듈 설치

$npm install axios cheerio

1) axios - npm (npmjs.com)

2) cheerio - npm (npmjs.com)

crawling.js

const axios = require("axios");
const cheerio = require("cheerio");

const getHTML = async (keyword) => {
    try {
        return await axios.get("https://www.inflearn.com/courses?s=" + encodeURI(keyword));
    } catch (error) {
        console.log(error);
    }
}


const parsing = async (keyword) => {
    const html = await getHTML(keyword);
    //console.log(html);
    const $ = cheerio.load(html.data);
    //jquery 처럼 사용할 수 있음
    const $courseList = $(".course_card_item");

    let courses = [];
    $courseList.each((idx, node) => {

        courses.push({
            title: $(node).find(".course_title:eq(0)").text(),
            instructor: $(node).find(".instructor").text(),
            price: $(node).find(".price").text(),
            rating: $(node).find(".star_solid").css("width"),
            img: $(node).find(".card-image > figure > img").attr("src")
        });

    });
    console.log(courses);
}

parsing("자바스크립트");

출력:

[
  {
    title: '하루만에 Cypress로 작성하는 자바스크립트 E2E 테스트 코드',
    instructor: '메이커준',
    price: '₩11,000₩7,700',
    rating: '0%',
    img: 'https://cdn.inflearn.com/public/courses/327991/cover/e3589cbb-7502-4c72-a665-72ec33d59173/인프런 썸네일.png'
  },
  {
    title: '자바스크립트 제대로 배워볼래?',
    instructor: '개발자의 품격',
    price: '₩77,000',
    rating: '98.23529411764707%',
    img: 'https://cdn.inflearn.com/public/courses/326778/cover/fa701aa5-8e85-4c60-8bd0-32be8a67635b/3A9329DD-8AC3-44FA-AEAA-F981219DC5E2.png'
  },
  {
    title: '입문자를 위한 자바스크립트 기초 강의',
    instructor: '유노코딩',
    price: '₩22,000',
    rating: '0%',
    img: 'https://cdn.inflearn.com/public/courses/327871/cover/5da57e4f-7cf0-416d-baad-3bf7758eb725/327871-eng.png'
  },
  {
    title: '풀스택을 위한 탄탄한 프런트엔드 부트캠프 (HTML, CSS, 바닐라 자바스크립트 + ES6) [풀스택 Part2]',
    instructor: '잔재미코딩',
    price: '₩71,500',
    rating: '98.125%',
    img: 'https://cdn.inflearn.com/public/course-326306-cover/4ea213de-f786-4af1-a9d0-b8932263885a'
  },
  {
    title: '자바스크립트 알고리즘 문제풀이(코딩테스트 대비)',
    instructor: '김태원',
    price: '₩60,500',
    rating: '97.89473684210526%',
    img: 'https://cdn.inflearn.com/public/courses/326377/cover/06b7fdbf-e58c-4289-b17d-bffddb67e999'
  },
  {
    title: '자바스크립트 비기너: 튼튼한 기본 만들기',
    instructor: '김영보',
    price: '₩44,000',
    rating: '95.04950495049505%',
    img: 'https://cdn.inflearn.com/public/course-324235-cover/12a6aceb-1c38-4ce1-b50c-ab9d32e43edd'
  },
  {
    title: '자바스크립트 머신러닝 TensorFlow.js',
    instructor: '김영보',
    price: '₩132,000',
    rating: '92%',
    img: 'https://cdn.inflearn.com/public/courses/324920/course_cover/7e7eb0b1-f56e-422a-b423-0464f1711feb/kyb-tensorFlow.jpg'
  },
  {
    title: '자바스크립트 ES6+ 기본',
    instructor: '김영보',
    price: '₩55,000',
    rating: '96.09756097560975%',
    img: 'https://cdn.inflearn.com/public/courses/324642/course_cover/3416cd43-b75d-422d-bc0f-ff6c9388f5ba/kyb_es6.png'
  },
  {
    title: '자바스크립트 중고급: 근본 핵심 이해',
    instructor: '김영보',
    price: '₩55,000',
    rating: '94.32432432432432%',
    img: 'https://cdn.inflearn.com/public/courses/324398/course_cover/97632ccf-eed2-4e51-baec-ef3fb81c9b38/kyb_advanced.jpg'
  },
  {
    title: '자바스크립트 언어 기본 - javascript',
    instructor: 'Egoing Lee',
    price: '무료',
    rating: '97.72151898734178%',
    img: 'https://cdn.inflearn.com/wp-content/uploads/js.png'
  },
  {
    title: '자바스크립트 ES6+ 심화',
    instructor: '김영보',
    price: '₩55,000',
    rating: '94.44444444444444%',
    img: 'https://cdn.inflearn.com/public/course-325633-cover/ececd36d-e3c5-49de-ac84-6bd44b8bb706'
  },
  {
    title: '오픈 소스 자바스크립트 React 프로그래밍 입문 Part.3',
    instructor: '유용한IT학습',
    price: '₩38,500₩26,950',
    rating: '0%',
    img: 'https://cdn.inflearn.com/public/courses/328022/cover/896d1cc2-9c2e-4b75-ad58-add567f5d365/328022-eng.png'
  },
  {
    title: '모던 자바스크립트 딥다이브 스터디',
    instructor: '정재남',
    price: '무료',
    rating: '0%',
    img: 'https://cdn.inflearn.com/public/courses/327974/cover/3b014384-8b3e-4f66-a4de-a94ffff11f58/Modern Javascript Deep Dive.png'
  },
  {
    title: '자바스크립트 입문자를 위한 강의',
    instructor: 'JeongTaeHyun',
    price: '₩27,500₩22,000',
    rating: '0%',
    img: 'https://cdn.inflearn.com/public/courses/326561/cover/77b01d30-3e19-4314-ad6e-4f9282ebfe8a'
  },
  {
    title: '빠르게 배우는 ES6 스킬과 자바스크립트 비동기 프로그래밍',
    instructor: '김형태',
    price: '₩9,900',
    rating: '91.53846153846153%',
    img: 'https://cdn.inflearn.com/wp-content/uploads/ES6-1-2.jpg'
  },
  {
    title: '오픈 소스 자바스크립트 React 프로그래밍 입문 Part.2',
    instructor: '유용한IT학습',
    price: '₩29,700₩20,790',
    rating: '0%',
    img: 'https://cdn.inflearn.com/public/courses/328021/cover/d2cd7bec-a069-4bbd-a425-ac41c32a9f74/328021-eng.png'
  },
  {
    title: '오픈 소스 자바스크립트 React 프로그래밍 입문 Part.1',
    instructor: '유용한IT학습',
    price: '₩38,500₩26,950',
    rating: '0%',
    img: 'https://cdn.inflearn.com/public/courses/328018/cover/fb6c713e-bbfb-4dca-b50f-e71f97cbd313/328018-eng.png'
  },
  {
    title: '함수형 프로그래밍과 JavaScript ES6+',
    instructor: '유인동',
    price: '₩55,000',
    rating: '98%',
    img: 'https://cdn.inflearn.com/wp-content/uploads/es6_8.jpg'
  },
  {
    title: '함수형 프로그래밍과 JavaScript ES6+ 응용편',
    instructor: '유인동',
    price: '₩44,000',
    rating: '99.68253968253968%',
    img: 'https://cdn.inflearn.com/public/courses/324019/course_cover/9bdc6818-f3d4-49f4-a0bf-1165e3f726b5/es6_2.png'
  },
  {
    title: 'Javascript ES6+ 제대로 알아보기 - 중급',
    instructor: '정재남',
    price: '₩49,500',
    rating: '98.70967741935485%',
    img: 'https://cdn.inflearn.com/wp-content/uploads/jjn_es62.jpg'
  },
  {
    title: '실습 UI 개발로 배워보는 순수 javascript 와 VueJS 개발',
    instructor: '김정환',
    price: '₩44,000',
    rating: '95.88235294117646%',
    img: 'https://cdn.inflearn.com/wp-content/uploads/vuejsandpure.png'
  },
  {
    title: 'Javascript ES6+ 제대로 알아보기 - 초급',
    instructor: '정재남',
    price: '₩35,200',
    rating: '98.55670103092784%',
    img: 'https://cdn.inflearn.com/wp-content/uploads/es6-1.jpg'
  },
  {
    title: '윤재성의 처음시작하는 JavaScript Tutorials',
    instructor: '소프트캠퍼스',
    price: '₩16,500',
    rating: '95%',
    img: 'https://cdn.inflearn.com/wp-content/uploads/sc_JS.jpg'
  },
  {
    title: '생활코딩 - JavaScript Immutability',
    instructor: 'Egoing Lee',
    price: '무료',
    rating: '98.82352941176471%',
    img: 'https://cdn.inflearn.com/public/courses/324803/course_cover/3cbd8ce1-4bdc-4302-9438-7b53c72bb125/egoing-js-immutability.png'
  }
]

Puppeteer  - puppeteer 사용법 로그인 후 정보 가져오기

puppeteer에 대한 매뉴얼 

https://pptr.dev/#?product=Puppeteer&version=v1.12.2&show=outline

puppeteer로 로그인 후 정보 가져오기

Crawling.js

var express = require('express');
var router = express.Router();

var puppeteer = require('puppeteer');

(async () => {

    const browser = await puppeteer.launch();
    const page = await browser.newPage();
    const hisnet_id = 'test@gmail.com';
    const hisnet_pw = 'IjmnfesfesfsQ4efeXKtWxa';


    //페이지로 가라
    await page.goto('http://macaronics.net/index.php/auth');

    //아이디랑 비밀번호 란에 값을 넣어라
    await page.evaluate((id, pw) => {
        document.querySelector('input[name="userid"]').value = id;
        document.querySelector('input[name="password"]').value = pw;
    }, hisnet_id, hisnet_pw);

    //로그인 버튼을 클릭해라
    await page.click('.btn-login-submit');

    console.log("로그인 버튼을 클릭해라");
    //로그인 화면이 전환될 때까지 3초만 기다려라
    await page.waitFor(3000);

    //로그인 실패시(화면 전환 실패시)
    if (page.url() === 'http://macaronics.net/index.php/auth') {
        console.log("로그인 실패");
        student_id = 'nope';
        name = 'nope';
    }
    //로그인 성공시
    else {
        console.log("로그인 성공");
        //다음 페이지로 가서
        await page.goto('http://macaronics.net/index.php/m04/mysql/view/1487');
        //post_content 내용을 가져오고
        const element1 = await page.$('.post_content');
        student_id = await page.evaluate(element1 => element1.textContent, element1);
        console.log(student_id);
        //small을 가져와라
        const element2 = await page.$('.post_date small');
        name = await page.evaluate(element2 => element2.textContent, element2);

        console.log(name);


        /* GET home page. */
        router.get('/', function (req, res, next) {
            res.render('crawling', { student_id: student_id });
        });


    }
    //브라우저 꺼라
    await browser.close();
})();





module.exports = router;

crawling.jade

extends layout

block content
  h1= title
  p Welcome to1  #{student_id}

.

https://zoomkoding.github.io/web/nodejs/histime/2019/01/24/crawler.html