DE Project 1 - ETL - 環境設定

 以下為所有 Docker Container 設定.

• Dockerfile
• Server採用 Ubuntu 20.04 版本
• 預安裝 Python 3.8
• 預安裝 Python packages, 連結 Database (MySQL & PostgreSQL)

 FROM ubuntu:20.04
 ARG DEBIAN_FRONTEND=noninteractive
 
 WORKDIR /usr/src/app
 
 RUN apt-get update && \
    apt-get install -y sudo && \
    apt-get install -y iputils-ping && \
    apt-get install -y --no-install-recommends apt-utils && \
    apt-get install -y software-properties-common && \
    add-apt-repository ppa:deadsnakes/ppa && \
    apt-get install python3.8 -y && \
    apt-get update -y && \
    apt-get install -y pkg-config python3.8-dev libmysqlclient-dev build-essential && \
    apt-get install -y git && \
    apt-get install -y python3-pip && \
    apt-get -y install unixodbc-dev && \
    python3.8 -m pip install pip --upgrade && \
    python3.8 -m pip install psycopg2-binary && \
    python3.8 -m pip install mechanize && \
    python3.8 -m pip install bs4 && \
    python3.8 -m pip install cryptography && \
    python3.8 -m pip install pycryptodomex && \
    apt-get install -y python-setuptools && \
    apt-get install -y libssl-dev && \
    apt-get install -y libffi-dev && \
    apt-get install -y libkrb5-dev && \
    apt-get install -y postgresql-client -y && \
    apt-get install curl -y && \
    apt-get install vim -y && \
    apt-get install python3.8-venv -y && \
    apt-get install -y locales && \
    apt-get install mysql-client -y && \
    apt-get install postgresql-client -y && \
    apt-get install python3-mysqldb -y && \
    apt-get update && \
    pip3 install sqlfluff && \
    pip3 install --upgrade pip && \
    pip3 install mysqlclient && \
    pip3 install Flask-MySQLdb mysql-connector-python
 
 WORKDIR /app/de

• docker-compose.yml
• 啟動四個服務: 
• Server: Ubuntu 20.04
• Warehousing DB: PostgresSQL
• AirFlowDB: MariaDB
• MySQL: MariaDB

version: '2.0'
services:
  ubuntu:
    build:
      context: .
      dockerfile: Dockerfile
    image: ubuntu:20.04
    container_name: de-2024
    volumes:
      - "../de_project_1/src/main:/app/de"
    command: /bin/bash
    stdin_open: true
    tty: true
    ports:
      - "8080:8080"
    links:
      - database
      - airflow
      - mysqldb
 
  database:
    image: postgres:14.0
    container_name: de-2024-pgdb
    environment:
      POSTGRES_USER: admin
      POSTGRES_PASSWORD: admin
      PGDATA: /var/lib/postgresql/data/pgdata
    volumes:
      - "../postgres:/var/lib/postgresql/data"
      - "../logs/db/logs:/var/log"
    ports:
      - "5432:5432"
 
  airflow:
    image: mariadb:10.5.8
    container_name: de-2024-afdb
    environment:
      MYSQL_ROOT_PASSWORD: root
      MYSQL_DATABASE: de
      MYSQL_USER: admin
      MYSQL_PASSWORD: admin
    volumes:
      - "../mysql:/var/lib/mysql"
      - "../mysql/data/conf:/etc/mysql/conf.d"
      - "../logs/db/logs:/var/log"
    ports:
      - "3308:3306"
 
  mysqldb:
    image: mariadb:11.3.1
    container_name: de-2024-mydb
    environment:
      MYSQL_ROOT_PASSWORD: root
      MYSQL_DATABASE: de
      MYSQL_USER: admin
      MYSQL_PASSWORD: admin
    volumes:
      - "../mariadb:/var/lib/mysql"
      - "../mariadb/data/conf:/etc/mysql/conf.d"
      - "../mariadb/logs/db/logs:/var/log"
    ports:
      - "3306:3306"

Docker containers