nice-crawler/index.js

import fetch from 'node-fetch';
import postgres from 'postgres';
import {drizzle} from 'drizzle-orm/postgres-js';
import {pgTable, text} from 'drizzle-orm/pg-core';

// PostgreSQL 데이터베이스 연결 설정
const sql = postgres({
    host: '172.30.1.93',   // PostgreSQL 호스트
    port: 5432,            // 포트 (기본값: 5432)
    database: 'bizinfo',   // 데이터베이스 이름
    username: 'postgres',  // 사용자 이름
    password: 'Dlstjq0904!' // 비밀번호
});

const orm = drizzle(sql); // Drizzle ORM 초기화

// PostgreSQL nice 테이블 정의
const niceTable = pgTable('nice', {
    empnum_bse_date: text('empnum_bse_date'),
    zcd3: text('zcd3'),
    bnknm: text('bnknm'),
    btpnm: text('btpnm'),
    etl_ipc_yn: text('etl_ipc_yn'),
    upt_date: text('upt_date'),
    korreprnm: text('korreprnm'),
    nolt_sgng: text('nolt_sgng'),
    zcd2: text('zcd2'),
    dtlcont: text('dtlcont'),
    crpno_rsdrgsbdtsexvl: text('crpno_rsdrgsbdtsexvl'),
    nolt_sido3: text('nolt_sido3'),
    chulja: text('chulja'),
    etb_date: text('etb_date'),
    nolt_sido2: text('nolt_sido2'),
    eqrObl: text('eqrObl'),
    slandc: text('slandc'),
    tel: text('tel'),
    fax: text('fax'),
    scl: text('scl'),
    zipareacdseq3: text('zipareacdseq3'),
    zipareacdseq2: text('zipareacdseq2'),
    nolt_sgng2: text('nolt_sgng2'),
    fadivcd: text('fadivcd'),
    sventure: text('sventure'),
    nolt_emdg: text('nolt_emdg'),
    zarcd: text('zarcd'),
    opt_entrnm: text('opt_entrnm'),
    empnum: text('empnum'),
    kor_itemnm: text('kor_itemnm'),
    upt_dtm: text('upt_dtm'),
    mainupche: text('mainupche'),
    nolt_engaddr2: text('nolt_engaddr2'),
    nolt_engaddr3: text('nolt_engaddr3'),
    nolt_dtl_koraddr: text('nolt_dtl_koraddr'),
    zipareacdseq: text('zipareacdseq'),
    nolt_sgng3: text('nolt_sgng3'),
    eng_idscdnm: text('eng_idscdnm'),
    tel2: text('tel2'),
    engentrnm: text('engentrnm'),
    crprgrnstscd: text('crprgrnstscd'),
    rbtpnm: text('rbtpnm'),
    engaddr3: text('engaddr3'),
    homepurl: text('homepurl'),
    tel3: text('tel3'),
    engaddr2: text('engaddr2'),
    amnisuyn: text('amnisuyn'),
    sido: text('sido'),
    bzdnm: text('bzdnm'),
    eng_scl: text('eng_scl'),
    zcd: text('zcd'),
    nolt_sido: text('nolt_sido'),
    repr_regno: text('repr_regno'),
    scaledivcd: text('scaledivcd'),
    eqrOblEng: text('eqrOblEng'),
    cls_yndivcd: text('cls_yndivcd'),
    zarcd3: text('zarcd3'),
    zarcd2: text('zarcd2'),
    nolt_dtl_koraddr2: text('nolt_dtl_koraddr2'),
    nolt_dtl_koraddr3: text('nolt_dtl_koraddr3'),
    idscdid: text('idscdid'),
    eng_mainupche: text('eng_mainupche'),
    eprdtldivcd: text('eprdtldivcd'),
    fa_bse_date: text('fa_bse_date'),
    sbqc_date: text('sbqc_date'),
    koraddr: text('koraddr'),
    koraddr3: text('koraddr3'),
    nolt_koraddr: text('nolt_koraddr'),
    koraddr2: text('koraddr2'),
    ltg_date: text('ltg_date'),
    rbtpnm3: text('rbtpnm3'),
    nolt_koraddr3: text('nolt_koraddr3'),
    nolt_koraddr2: text('nolt_koraddr2'),
    x: text('x'),
    y: text('y'),
    rdnm_sido2: text('rdnm_sido2'),
    hupegbn: text('hupegbn'),
    kisreporturl: text('kisreporturl'),
    rdnm_sido3: text('rdnm_sido3'),
    rbtpnm2: text('rbtpnm2'),
    mainpdtpcl: text('mainpdtpcl'),
    stkcd: text('stkcd'),
    bizno: text('bizno'),
    grpnm: text('grpnm'),
    sbn_date: text('sbn_date'),
    eprmdydivcd: text('eprmdydivcd'),
    eprdatastsdivcd: text('eprdatastsdivcd'),
    bnk_brnm: text('bnk_brnm'),
    korentrnm: text('korentrnm'),
    fax3: text('fax3'),
    logo: text('logo'),
    fax2: text('fax2'),
    rdnm_dtl_koraddr: text('rdnm_dtl_koraddr'),
    rdnm_sgng3: text('rdnm_sgng3'),
    rdnm_sido: text('rdnm_sido'),
    eng_grpnm: text('eng_grpnm'),
    nts_sbqcdivcd: text('nts_sbqcdivcd'),
    eng_mainpdtpcl: text('eng_mainpdtpcl'),
    kiscode: text('kiscode'),
    korreprcd: text('korreprcd'),
    engaddr: text('engaddr'),
    rdnm_sgng2: text('rdnm_sgng2'),
    smanda: text('smanda'),
    ltgmktdivcd: text('ltgmktdivcd'),
    obz_date: text('obz_date'),
    x2: text('x2'),
    eng_itemnm: text('eng_itemnm'),
    x3: text('x3'),
    fadivnm: text('fadivnm'),
    sforeign: text('sforeign'),
    eml: text('eml'),
    crpno: text('crpno'),
    rdnm_sgng: text('rdnm_sgng'),
    y2: text('y2'),
    nolt_emdg3: text('nolt_emdg3'),
    y3: text('y3'),
    nolt_emdg2: text('nolt_emdg2'),
    korrdnm: text('korrdnm'),
    eng_btpnm: text('eng_btpnm'),
    bzdnm2: text('bzdnm2'),
    bzdnm3: text('bzdnm3'),
    kor_idscdnm: text('kor_idscdnm'),
    eng_bnknm: text('eng_bnknm'),
    logossl: text('logossl'),
    stacmm: text('stacmm'),
    idscd: text('idscd'),
    repr_rsdrgsbdtsexvl: text('repr_rsdrgsbdtsexvl'),
    gicd: text('gicd'),
    epr_cnu_yn: text('epr_cnu_yn'),
    nolt_engaddr: text('nolt_engaddr'),
    korrdnm3: text('korrdnm3'),
    korrdnm2: text('korrdnm2'),
    engreprnm: text('engreprnm'),
    rdnm_dtl_koraddr3: text('rdnm_dtl_koraddr3'),
    rdnm_dtl_koraddr2: text('rdnm_dtl_koraddr2')
});

async function main() {
    const baseUrl = 'https://nice.kegorii.workers.dev/get?t=';
    const firstArray = ['S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z'];
    for (let i = 0; i < firstArray.length; i++) {
        for (let y = 0; y < 10000; y++) {
            const rtnval = y.toString().padStart(4, '0');
            const kiscode = firstArray[i]  + rtnval;
            const url = `${baseUrl}${kiscode}`;
            try {
                const startTime = Date.now(); // 시작 시간 저장

                const response = await fetch(url);

                if (!response.ok) {
                    throw new Error(`Request failed: ${response.statusText}`);
                }

                const jsonData = await response.json();
                const insertValues = jsonData.map(item => ({
                    ...item // 모든 데이터를 그대로 삽입 (데이터 정제 필요)
                }));

                if (insertValues.length > 0) {
                    await orm.insert(niceTable).values(insertValues);
                }

                const endTime = Date.now(); // 종료 시간 저장
                console.log(`Execution time for kiscode:${kiscode}: ${endTime - startTime} ms`); // 콘솔에 시간 출력
            } catch (error) {
                console.error(`Error at kiscode:${kiscode}`, error);
            }
        }
    }

}

main();