diff --git a/.gitignore b/.gitignore index 8b10abf..8057aaf 100644 --- a/.gitignore +++ b/.gitignore @@ -34,4 +34,6 @@ lerna-debug.log* !.vscode/launch.json !.vscode/extensions.json -**/.env \ No newline at end of file +**/.env + +src/public/data/**/*.json \ No newline at end of file diff --git a/nest-cli.json b/nest-cli.json index 56167b3..9f38d55 100644 --- a/nest-cli.json +++ b/nest-cli.json @@ -1,4 +1,12 @@ { "collection": "@nestjs/schematics", - "sourceRoot": "src" + "sourceRoot": "src", + "compilerOptions": { + "assets": [ + { + "include": "./public/data/result/*.json", + "outDir": "./dist" + } + ] + } } diff --git a/package-lock.json b/package-lock.json index 743f322..6e57d1e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -16,9 +16,11 @@ "@nestjs/platform-express": "^8.0.0", "@nestjs/swagger": "^5.2.0", "@nestjs/typeorm": "^8.0.3", + "axios": "^0.26.0", "class-transformer": "^0.5.1", "class-validator": "^0.13.2", "cross-env": "^7.0.3", + "dotenv": "^16.0.0", "express-basic-auth": "^1.2.1", "firebase-admin": "^10.0.1", "joi": "^17.6.0", diff --git a/package.json b/package.json index 5b60d1c..7b14469 100644 --- a/package.json +++ b/package.json @@ -27,9 +27,11 @@ "@nestjs/platform-express": "^8.0.0", "@nestjs/swagger": "^5.2.0", "@nestjs/typeorm": "^8.0.3", + "axios": "^0.26.0", "class-transformer": "^0.5.1", "class-validator": "^0.13.2", "cross-env": "^7.0.3", + "dotenv": "^16.0.0", "express-basic-auth": "^1.2.1", "firebase-admin": "^10.0.1", "joi": "^17.6.0", diff --git a/src/app.module.ts b/src/app.module.ts index af24cfe..dbe7dc1 100644 --- a/src/app.module.ts +++ b/src/app.module.ts @@ -4,6 +4,7 @@ import { validationSchema } from './config/validationSchema'; import { TypeOrmModule, TypeOrmModuleOptions } from '@nestjs/typeorm'; import { UserModule } from './user/user.module'; import { PostModule } from './post/post.module'; +import { MountainModule } from './mountain/mountain.module'; @Module({ imports: [ @@ -28,6 +29,7 @@ import { PostModule } from './post/post.module'; }), UserModule, PostModule, + MountainModule, ], }) export class AppModule {} diff --git a/src/database/entities/mountains.entity.ts b/src/database/entities/mountains.entity.ts index a93d3c7..7ee7217 100644 --- a/src/database/entities/mountains.entity.ts +++ b/src/database/entities/mountains.entity.ts @@ -17,10 +17,10 @@ export class MountainsEntity extends DateAuditEntity { @Column() url: string; - @Column() + @Column({ type: 'numeric' }) lat: number; - @Column() + @Column({ type: 'numeric' }) lon: number; @Column('int', { name: 'imageId', nullable: true }) diff --git a/src/lib/mountain-generator/index.js b/src/lib/mountain-generator/index.js new file mode 100644 index 0000000..aae0467 --- /dev/null +++ b/src/lib/mountain-generator/index.js @@ -0,0 +1,129 @@ +const REGION_BASE = require('./region_base'); + +const axios = require('axios'); +const fs = require('fs-extra'); + +const path = require('path'); +require('dotenv').config(); + +const targetPath = path.join(__dirname, '..', '..'); + +const KAKAO_REST_API_KEY = process.env.KAKAO_REST_API_KEY; + +const updatedAt = Date.now(); + +let totalDataCount = 0; + +const resultMap = new Map(); + +const uniquenessMap = new Map(); + +const regionKeys = Object.keys(REGION_BASE); + +const getTargetPath = (region) => `${targetPath}/public/data/result/${region}.json`; + +const TARGET_DIR = `${targetPath}/public/data/result`; + +const LOCATION_PATH = `${targetPath}/public/data/result/location.json`; + +const RESULT_PATH = `${targetPath}/public/data/result/total_count.txt`; + +const sleep = (time) => new Promise((resolve) => setTimeout(resolve, time)); + +// 검색어, 카테고리 // +const QUERY = '산'; +const CATEGORY_NAME = '여행 > 관광,명소 > 산'; + +// ex) ['서울특별시 종로구', ...]; +const REGION = Object.entries(REGION_BASE).reduce((acc, [k, v]) => [...acc, ...v.map((u) => `${k} ${u}`)], []); + +const kakaoSearchAPI = ({ query, page = 1 }) => + axios.get(`https://dapi.kakao.com/v2/local/search/keyword.json`, { + headers: { + Authorization: `KakaoAK ${KAKAO_REST_API_KEY}`, + }, + params: { + query, + page, + }, + timeout: 10000, + }); + +async function* asyncAPI(query, total_pages) { + for (let i = 1; i <= total_pages; i++) { + const { data } = await kakaoSearchAPI({ query, page: i }); + await sleep(300); + yield data; + } +} + +const fetchRegionData = async () => { + for (const region of REGION) { + const query = `${region} ${QUERY}`; + const regionKey = region.split(' ')[0]; + const targetRegionArray = resultMap.get(regionKey); + + const { + data: { meta }, + } = await kakaoSearchAPI({ query }); + + const { pageable_count, total_count } = meta; + + if (pageable_count * 15 < total_count) { + throw new Error(`${region} 지역 정보는 더 상세한 검색이 필요합니다`); + } + + for await (const { documents } of asyncAPI(query, pageable_count)) { + documents.forEach((v) => { + const isDuplicatePresent = uniquenessMap.has(v.id); + if (isDuplicatePresent) return; + uniquenessMap.set(v.id, true); + + const isCategoryMatch = v.category_name !== CATEGORY_NAME; + if (isCategoryMatch) return; + + targetRegionArray.push(v); + }); + } + + console.log(`${region} 지역 데이터를 성공적으로 생성 하였습니다.`); + } +}; + +const writeRegionData = () => { + for (const regionKey of regionKeys) { + const regionResultArray = resultMap.get(regionKey); + + totalDataCount += regionResultArray.length; + + fs.writeFileSync(getTargetPath(regionKey), JSON.stringify(regionResultArray)); + } +}; + +const writeResult = () => { + const endAt = new Date().getTime(); + fs.writeFileSync(RESULT_PATH, `totalDataCount : ${totalDataCount} time: ${(endAt - updatedAt) / 1000}`); + fs.writeFileSync(LOCATION_PATH, JSON.stringify(REGION_BASE)); +}; + +(async () => { + if (!fs.existsSync(TARGET_DIR)) { + fs.mkdirSync(TARGET_DIR); + } + + for (const regionKey of regionKeys) { + resultMap.set(regionKey, []); + } + + try { + await fetchRegionData(); + + writeRegionData(); + + writeResult(); + console.log('모든 데이터를 성공적으로 생성 하였습니다'); + } catch (error) { + console.warn('전체 데이터를 생성하는 중 문제가 발생 했습니다', error); + fs.rmdirSync(TARGET_DIR, { recursive: true }); + } +})(); diff --git a/src/lib/mountain-generator/region_base.js b/src/lib/mountain-generator/region_base.js new file mode 100644 index 0000000..1d217c4 --- /dev/null +++ b/src/lib/mountain-generator/region_base.js @@ -0,0 +1,224 @@ +const REGION_BASE = { + 서울특별시: [ + '종로구', + '중구', + '용산구', + '성동구', + '광진구', + '동대문구', + '중랑구', + '성북구', + '강북구', + '도봉구', + '노원구', + '은평구', + '서대문구', + '마포구', + '양천구', + '강서구', + '구로구', + '금천구', + '영등포구', + '동작구', + '관악구', + '서초구', + '강남구', + '송파구', + '강동구', + ], + 인천광역시: ['중구', '동구', '미추홀구', '연수구', '남동구', '부평구', '계양구', '서구', '강화군', '옹진군'], + 울산광역시: ['중구', '남구', '동구', '북구', '울주군'], + 경기도: [ + '수원시', + '성남시', + '고양시', + '용인시', + '부천시', + '안산시', + '안양시', + '남양주시', + '화성시', + '평택시', + '의정부시', + '시흥시', + '파주시', + '광명시', + '김포시', + '군포시', + '광주시', + '이천시', + '양주시', + '오산시', + '구리시', + '안성시', + '포천시', + '의왕시', + '하남시', + '여주시', + '양평군', + '동두천시', + '과천시', + '가평군', + '연천군', + ], + 강원도: [ + '춘천시', + '원주시', + '강릉시', + '동해시', + '태백시', + '속초시', + '삼척시', + '홍천군', + '횡성군', + '영월군', + '평창군', + '정선군', + '철원군', + '화천군', + '양구군', + '인제군', + '고성군', + '양양군', + ], + 세종특별자치시: ['세종특별자치시'], + 충청남도: [ + '천안시', + '공주시', + '보령시', + '아산시', + '서산시', + '논산시', + '계룡시', + '당진시', + '금산군', + '부여군', + '서천군', + '청양군', + '홍성군', + '예산군', + '태안군', + ], + 대전광역시: ['동구', '중구', '서구', '유성구', '대덕구'], + 충청북도: [ + '청주시', + '충주시', + '제천시', + '보은군', + '옥천군', + '영동군', + '진천군', + '괴산군', + '음성군', + '단양군', + '증평군', + ], + 경상북도: [ + '포항시', + '경주시', + '김천시', + '안동시', + '구미시', + '영주시', + '영천시', + '상주시', + '문경시', + '경산시', + '군위군', + '의성군', + '청송군', + '영양군', + '영덕군', + '청도군', + '고령군', + '성주군', + '칠곡군', + '예천군', + '봉화군', + '울진군', + '울릉군', + ], + 대구광역시: ['중구', '동구', '서구', '남구', '북구', '수성구', '달서구', '달성군'], + 전라북도: [ + '전주시', + '군산시', + '익산시', + '정읍시', + '남원시', + '김제시', + '완주군', + '진안군', + '무주군', + '장수군', + '임실군', + '순창군', + '고창군', + '부안군', + ], + 광주광역시: ['동구', '서구', '남구', '북구', '광산구'], + 전라남도: [ + '목포시', + '여수시', + '순천시', + '나주시', + '광양시', + '담양군', + '곡성군', + '구례군', + '고흥군', + '보성군', + '화순군', + '장흥군', + '강진군', + '해남군', + '영암군', + '무안군', + '함평군', + '영광군', + '장성군', + '완도군', + '진도군', + '신안군', + ], + 경상남도: [ + '창원시', + '진주시', + '통영시', + '사천시', + '김해시', + '밀양시', + '거제시', + '양산시', + '의령군', + '함안군', + '창녕군', + '고성군', + '남해군', + '하동군', + '산청군', + '함양군', + '거창군', + '합천군', + ], + 부산광역시: [ + '중구', + '서구', + '동구', + '영도구', + '부산진구', + '동래구', + '남구', + '북구', + '해운대구', + '사하구', + '금정구', + '강서구', + '연제구', + '수영구', + '사상구', + '기장군', + ], + 제주특별자치도: ['제주시', '서귀포시'], +}; + +module.exports = REGION_BASE; diff --git a/src/mountain/mountain.helper.ts b/src/mountain/mountain.helper.ts new file mode 100644 index 0000000..679c685 --- /dev/null +++ b/src/mountain/mountain.helper.ts @@ -0,0 +1,51 @@ +import { Injectable, Logger, OnModuleInit } from '@nestjs/common'; +import { InjectRepository } from '@nestjs/typeorm'; +import { MountainsEntity } from 'src/database/entities/mountains.entity'; +import { Repository } from 'typeorm'; + +export interface IMountainInforamtion { + address_name: string; + place_url: string; + place_name: string; + x: number; + y: number; +} + +@Injectable() +export class MountainHelper implements OnModuleInit { + constructor(@InjectRepository(MountainsEntity) private readonly mountainRepository: Repository) {} + async onModuleInit() { + if (!(await this.isDataBinding())) { + Logger.log('DB에 산 데이터가 없습니다. 저장을 시작합니다.', 'MountainHelper'); + await this.seedingData(); + } + } + + private async isDataBinding(): Promise { + const mountains = await this.mountainRepository.createQueryBuilder('mountains').limit(100).getMany(); + if (mountains.length === 0) { + return false; + } else return true; + } + private async seedingData(): Promise { + try { + let locations = require('../public/data/result/location.json'); + locations = Object.keys(locations); + locations.forEach((location: string) => { + const data: IMountainInforamtion[] = require(`../public/data/result/${location}.json`); + data.forEach(async (mountain: IMountainInforamtion) => { + await this.mountainRepository.save({ + name: mountain.place_name, + address: mountain.address_name, + url: mountain.place_url, + lat: mountain.y, + lon: mountain.x, + }); + }); + }); + Logger.log('산 데이터를 DB에 성공적으로 저장했습니다.', 'MountainHelper'); + } catch (err) { + Logger.log('JSON 데이터를 읽는데 실패했습니다.', 'MountainHelper'); + } + } +} diff --git a/src/mountain/mountain.module.ts b/src/mountain/mountain.module.ts new file mode 100644 index 0000000..c7855b0 --- /dev/null +++ b/src/mountain/mountain.module.ts @@ -0,0 +1,10 @@ +import { Module } from '@nestjs/common'; +import { TypeOrmModule } from '@nestjs/typeorm'; +import { MountainsEntity } from 'src/database/entities/mountains.entity'; +import { MountainHelper } from './mountain.helper'; + +@Module({ + imports: [TypeOrmModule.forFeature([MountainsEntity])], + providers: [MountainHelper], +}) +export class MountainModule {} diff --git a/src/post/post.module.ts b/src/post/post.module.ts index 116ff9b..9d59937 100644 --- a/src/post/post.module.ts +++ b/src/post/post.module.ts @@ -1,4 +1,5 @@ import { Module } from '@nestjs/common'; +import { CqrsModule } from '@nestjs/cqrs'; import { TypeOrmModule } from '@nestjs/typeorm'; import { PostMembers } from 'src/database/entities/postMembers.entity'; import { PostsEntity } from 'src/database/entities/posts.entity'; @@ -7,7 +8,7 @@ import { PostController } from './controller/post.controller'; import { PostService } from './services/post.service'; @Module({ - imports: [TypeOrmModule.forFeature([PostsEntity, PostMembers]), UserModule], + imports: [TypeOrmModule.forFeature([PostsEntity, PostMembers]), UserModule, CqrsModule], controllers: [PostController], providers: [PostService], }) diff --git a/src/public/data/result/total_count.txt b/src/public/data/result/total_count.txt new file mode 100644 index 0000000..d83ed90 --- /dev/null +++ b/src/public/data/result/total_count.txt @@ -0,0 +1 @@ +totalDataCount : 6121 time: 3653.657 \ No newline at end of file diff --git a/tsconfig.json b/tsconfig.json index adb614c..d2da3ec 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -1,6 +1,5 @@ { "compilerOptions": { - "module": "commonjs", "declaration": true, "removeComments": true, "emitDecoratorMetadata": true, @@ -16,6 +15,7 @@ "noImplicitAny": false, "strictBindCallApply": false, "forceConsistentCasingInFileNames": false, - "noFallthroughCasesInSwitch": false + "noFallthroughCasesInSwitch": false, + "module": "CommonJS" } }