commit 89e706d26bf2dc6c5608904c2c4f3ba3305e612c Author: Insub Kim Date: Tue Apr 29 00:13:10 2025 +0900 Add data fetching and storage script with PostgreSQL integration This commit introduces a script to fetch data from an external API, process it, and store it in a PostgreSQL database using Drizzle ORM. It sets up necessary dependencies in `package.json` and defines the database schema, connection, and logic to handle paginated API requests efficiently. diff --git a/index.js b/index.js new file mode 100644 index 0000000..5984f25 --- /dev/null +++ b/index.js @@ -0,0 +1,117 @@ +import fetch from 'node-fetch'; +import postgres from 'postgres'; +import {drizzle} from 'drizzle-orm/postgres-js'; +import {pgTable, text} from 'drizzle-orm/pg-core'; + +// PostgreSQL 데이터베이스 연결 설정 +const sql = postgres({ + host: '172.30.1.93', // PostgreSQL 호스트 + port: 5432, // 포트 (기본값: 5432) + database: 'bizinfo', // 데이터베이스 이름 + username: 'postgres', // 사용자 이름 + password: 'Dlstjq0904!'// 비밀번호 +}); + +const orm = drizzle(sql); // Drizzle ORM 초기화 + +// PostgreSQL 테이블 정의 +const jsonTable = pgTable('kreport', { + kedcd: text('kedcd'), + enpNmF: text('enpNmF'), + bzno: text('bzno'), + reperNm: text('reperNm'), + ipoCdNm: text('ipoCdNm'), + ipoCd: text('ipoCd'), + enpScd: text('enpScd'), + enpSzeNm: text('enpSzeNm'), + telNo: text('telNo'), + ksic11BzcCdNm: text('ksic11BzcCdNm'), + bzplAddrb: text('bzplAddrb'), + cono: text('cono'), + pid: text('pid'), + estbDt: text('estbDt'), + enpTyp: text('enpTyp'), + bzplSidoNm: text('bzplSidoNm'), + bzplGugunNm: text('bzplGugunNm'), + bzplDongNm: text('bzplDongNm'), + bzplRdnmSidoNm: text('bzplRdnmSidoNm'), + bzplRdnmGugunNm: text('bzplRdnmGugunNm'), + bzplRdNm: text('bzplRdNm') +}); + +async function main() { + const baseUrl = 'https://app.kegorii.workers.dev/get'; + for (let y = 2024; y < 2026; y++) { + for (let t = 1; t < 100; t++) { + const chkCountParam = {t: t, p: 1, y: y}; + const queryString = new URLSearchParams(chkCountParam).toString(); + const url = `${baseUrl}?${queryString}`; + const response = await fetch(url); + const jsonData = await response.json(); + const cmTotalRowCount = jsonData.cmTotalRowCount; + const totalPage = calculatePages(cmTotalRowCount, 10); + console.log(`t: ${t}`); + console.log(`cmTotalRowCount: ${cmTotalRowCount}`); + console.log(`totalPage: ${totalPage}`); + for (let p = 1; p < totalPage + 1; p++) { + try { + const queryParams = {t, p, y}; + const queryString = new URLSearchParams(queryParams).toString(); + + const url = `${baseUrl}?${queryString}`; + console.log(`생성된 url: ${url}`); + const response = await fetch(url); + + if (!response.ok) { + throw new Error(`네트워크 요청에 실패하였습니다: ${response.statusText}`); + } + + const jsonData = await response.json(); + + // jsonData.data는 배열 형태로 가정 + const insertValues = jsonData.data.map((item) => ({ + kedcd: item.kedcd, + enpNmF: item.enpNmF, + bzno: item.bzno, + reperNm: item.reperNm, + ipoCdNm: item.ipoCdNm, + ipoCd: item.ipoCd, + enpScd: item.enpScd, + enpSzeNm: item.enpSzeNm, + telNo: item.telNo, + ksic11BzcCdNm: item.ksic11BzcCdNm, + bzplAddrb: item.bzplAddrb, + cono: item.cono, + pid: item.pid, + estbDt: item.estbDt, + enpTyp: item.enpTyp, + bzplSidoNm: item.bzplSidoNm, + bzplGugunNm: item.bzplGugunNm, + bzplDongNm: item.bzplDongNm, + bzplRdnmSidoNm: item.bzplRdnmSidoNm, + bzplRdnmGugunNm: item.bzplRdnmGugunNm, + bzplRdNm: item.bzplRdNm + })); + + if (insertValues.length > 0) { + await orm + .insert(jsonTable) + .values(insertValues); // `run()` 생략 + console.log('데이터 저장완료:' + p); + } else { + console.log('데이터 없음. 저장과정 생략함.:' + p); + } + } catch (error) { + console.error('오류 발생:', error); + } + } + } + } + +} + +function calculatePages(totalCount, itemsPerPage) { + return Math.ceil(totalCount / itemsPerPage); +} + +main(); \ No newline at end of file diff --git a/package.json b/package.json new file mode 100644 index 0000000..1272481 --- /dev/null +++ b/package.json @@ -0,0 +1,19 @@ +{ + "name": "mgr", + "version": "1.0.0", + "main": "index.js", + "type": "module", + "scripts": { + "test": "echo \"Error: no test specified\" && exit 1" + }, + "author": "", + "license": "ISC", + "description": "", + "dependencies": { + "dotenv": "^16.5.0", + "drizzle-orm": "^0.42.0", + "node-fetch": "^3.3.2", + "pg": "^8.14.1", + "postgres": "^3.4.5" + } +}