From 85478c26dca4e59bfb9059ee0144650cfeb876c6 Mon Sep 17 00:00:00 2001 From: p23 Date: Tue, 17 Dec 2024 19:06:40 +0000 Subject: [PATCH] fix some bugs --- .gitignore | 5 +---- Dockerfile | 18 ++++++++++++++++++ README.md | 19 +++++++++++++++++++ app.py | 17 ++++++++++++++--- db/dbhelper.py | 1 - docker-compose.yml | 26 ++++++++++++++++++++++++++ grpcServer/anoth.py | 26 ++++++++++++++++++-------- grpcServer/grpcServer.py | 21 +++++++++++++-------- ig/IG.py | 22 ++++++++++++++-------- s3/s3helper.py | 9 +++++++-- utils/tbProcessor.py | 6 ++++-- 11 files changed, 134 insertions(+), 36 deletions(-) create mode 100644 Dockerfile create mode 100644 README.md create mode 100644 docker-compose.yml diff --git a/.gitignore b/.gitignore index 7a3fa2f..5541dd4 100644 --- a/.gitignore +++ b/.gitignore @@ -3,8 +3,5 @@ __pycache__ session.json traceback.json test.py - tmp -tmp/* -testFiles -testFiles/* \ No newline at end of file +tmp/* \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..ef24ede --- /dev/null +++ b/Dockerfile @@ -0,0 +1,18 @@ +FROM python:3.11 + +WORKDIR /app + +# apt install +RUN apt-get update && \ + DEBAIN_FRONTEND=noninteractive apt-get install -qy ffmpeg +# other packages if needed +# git openssh-server vim net-tools iputils-ping btop tmux wget + +# pip3 install +COPY ./requirements.txt /app/requirements.txt +COPY ./ffmpeg_python-0.2.0-py3-none-any.whl /app/ffmpeg_python-0.2.0-py3-none-any.whl + +RUN pip3 install -r /app/requirements.txt +RUN pip3 install ffmpeg_python-0.2.0-py3-none-any.whl + +EXPOSE 50051 diff --git a/README.md b/README.md new file mode 100644 index 0000000..b299934 --- /dev/null +++ b/README.md @@ -0,0 +1,19 @@ +# Niming Backend (IGAPI) + +## build +### with docker-compose +See ``docker-compose.yml`` (It is a template) + +### Manual +Prepare: +``` +pip3 install -r requirements.txt +pip3 install ffmpeg_python-0.2.0-py3-none-any.whl +``` + +Run: +``` +python3 app.py +``` + +Shirakami Fubuki is the cutest fox!!! diff --git a/app.py b/app.py index 66984bf..5d902df 100644 --- a/app.py +++ b/app.py @@ -2,6 +2,7 @@ import os import sys import asyncio import threading +import logging from sqlalchemy import create_engine from instagrapi import Client @@ -11,16 +12,26 @@ from ig import IG from db import dbhelper from db.pgclass import Base from grpcServer import grpcServer, anoth -from utils.const import DEBUG +from utils.const import DEBUG, TMP_DIR # load_dotenv() +# logging +logging.basicConfig( + level=logging.INFO, + format='%(asctime)s | [%(levelname)s] %(name)s - %(message)s' +) + if DEBUG: - print("[*] ===== DEBUG MODE =====") + logging.info("===== DEBUG MODE =====") + +# tmp dir +if not os.path.exists(TMP_DIR): + os.mkdir(TMP_DIR) # Database PG_HOST = os.environ.get("PG_HOST", None).strip() -print("[*] Connecting to Database") +logging.info("Connecting to Database") dbhelper.db = dbhelper.DB(create_engine(PG_HOST)) Base.metadata.create_all(dbhelper.db._engine) diff --git a/db/dbhelper.py b/db/dbhelper.py index 841c5a3..46a7ff6 100644 --- a/db/dbhelper.py +++ b/db/dbhelper.py @@ -146,7 +146,6 @@ def solo_article_set_igid(id:int, igid:str) -> int: stmt = update(article_meta).where(article_meta.hash==hash).values(igid=igid) session.execute(stmt) except Exception as e: - print(e) err = 1 session.commit() return err diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..c5078eb --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,26 @@ +version: '3' + +# template: docker-compose.yml + +services: + niming-backend-igrpc: + build: . + container_name: niming-backend-igrpc + volumes: + - "igrpc_data/:/app" + ports: + - "50051:50051" + - "10000:10000" # i dont know what is listening this but this is written in docker-compose(dev) + environment: + - PG_HOST=postgresql+psycopg2://root:password@ip:port/niming_db + - ACCOUNT_USERNAME= + - ACCOUNT_PASSWORD= + - S3_ENDPOINT=ip:port + - S3_ACCESS_KEY= + - S3_SECRET_KEY= + - S3_BUCKET=nmfs + restart: unless-stopped + working_dir: /app + command: python3 /app/app.py + #networks: + # - networkName diff --git a/grpcServer/anoth.py b/grpcServer/anoth.py index d9087cf..dbba5ab 100644 --- a/grpcServer/anoth.py +++ b/grpcServer/anoth.py @@ -1,22 +1,29 @@ import time import random +import logging from grpcServer import postProcessor from utils.ThreadSafeOrderedDict import ThreadSafeOrderedDict from utils.const import ANOTH_INTERVAL_MIN, ANOTH_INTERVAL_MAX +from utils.tbProcessor import easyExceptionHandler from db import dbhelper +# logging +anothlog = logging.getLogger("anoth") +anothlog.setLevel(level=logging.INFO) + +# task queue task = ThreadSafeOrderedDict() def task_round(): t = task.popitem(last=False) if not t: # 沒任務 - print("[*] No task in queue") + anothlog.info("No task in queue") return aid = t[1]["aid"] type = t[0].split("-")[0] - print("[*] Task %s(target_aid=%d)"%(type, aid)) + anothlog.info("Task %s(target_aid=%d)"%(type, aid)) if type == "upload": # upload msg, err = postProcessor.upload(aid) @@ -27,21 +34,24 @@ def task_round(): msg, err = "Invalid task type %s"%type, 1 if err: - print("[X] Task failed: %s"%msg) + anothlog.error("Task failed: %s"%msg) elif type == "upload": dberr = dbhelper.solo_article_set_igid(id=aid, igid=msg) if dberr: - print("[X] Task %s(target_aid=%d): Set igid failed"%(type, aid)) + anothlog.error("Task %s(target_aid=%d): Set igid failed"%(type, aid)) - print("[*] Task Done") + anothlog.info("Task Done") return def run(): - print("[*] Upload/Delete Processor Started") + anothlog.info("Upload/Delete Processor Started") while True: - task_round() + try: + task_round() + except Exception as e: + easyExceptionHandler(e) sleep = random.randint(ANOTH_INTERVAL_MIN, ANOTH_INTERVAL_MAX) - print("[*] Next Round After %ds"%sleep) + anothlog.info("Next Round After %ds"%sleep) time.sleep(sleep) diff --git a/grpcServer/grpcServer.py b/grpcServer/grpcServer.py index 1f519ac..825c44e 100644 --- a/grpcServer/grpcServer.py +++ b/grpcServer/grpcServer.py @@ -1,4 +1,5 @@ # from concurrent import futures +import logging import grpc from cachetools import cached, TTLCache @@ -10,6 +11,10 @@ from grpcServer import anoth from grpcServer.protobuf import igapi_pb2_grpc from grpcServer.protobuf.igapi_pb2 import Request, Reply +# logging +grpclog = logging.getLogger("grpc") +grpclog.setLevel(level=logging.INFO) + # call account info / login cache_accinfo = TTLCache(maxsize=1, ttl=GRPC_ACCINFO_CACHE) @cached(cache_accinfo) @@ -28,7 +33,7 @@ def call_IG_login(): # 考慮一下如果同時發起多的請求，asyncio可能會搞到被ban號(IG) class IGAPI_Server(igapi_pb2_grpc.IGAPIServicer): async def account_info(self, request: Request, context) -> Reply: - print("[*] Request: account_info") + grpclog.info("Request: account_info") account = call_IG_account_info() if account: result = { @@ -42,9 +47,9 @@ class IGAPI_Server(igapi_pb2_grpc.IGAPIServicer): async def login(self, request: Request, context) -> Reply: - print("[*] Request: login") + grpclog.info("Request: login") if len(cache_login): # cache has not expired - print("[*] Login: Cooldown") + grpclog.info("Login: Cooldown") return Reply(err=1, result={"error":"Cooldown"}) else: login = call_IG_login() @@ -55,7 +60,7 @@ class IGAPI_Server(igapi_pb2_grpc.IGAPIServicer): async def upload(self, request: Request, context) -> Reply: - print("[*] Request: upload") + grpclog.info("Request: upload") aid = request.code # 檢查 - 可見 @@ -83,7 +88,7 @@ class IGAPI_Server(igapi_pb2_grpc.IGAPIServicer): async def delete(self, request: Request, context) -> Reply: - print("[*] Request: delete") + grpclog.info("Request: delete") # article id aid = request.code # igid from args @@ -108,7 +113,7 @@ class IGAPI_Server(igapi_pb2_grpc.IGAPIServicer): async def queue(self, request:Request, context) -> Reply: - print("[*] Request: queue") + grpclog.info("Request: queue") t = anoth.task.items() reply = { _[0]:str(_[1]["aid"]) for _ in t } return Reply(err=0, result=reply) @@ -116,7 +121,7 @@ class IGAPI_Server(igapi_pb2_grpc.IGAPIServicer): async def setting(self, request:Request, context) -> Reply: # not done - print("[*] Request: setting") + grpclog.info("Request: setting") return Reply(err=1, result={"error":"Not Done"}) @@ -128,5 +133,5 @@ async def serve() -> None: ) server.add_insecure_port("[::]:50051") await server.start() - print("[*] gRPC Server listening on 0.0.0.0:50051") + grpclog.info("gRPC Server listening on 0.0.0.0:50051") await server.wait_for_termination() diff --git a/ig/IG.py b/ig/IG.py index fc5b9e1..7b66a00 100644 --- a/ig/IG.py +++ b/ig/IG.py @@ -1,4 +1,5 @@ import os +import logging from typing import List from instagrapi import Client @@ -6,6 +7,10 @@ from instagrapi import Client from utils.tbProcessor import easyExceptionHandler from utils.const import DEVICE +# logging +iglog = logging.getLogger("ig") +iglog.setLevel(level=logging.DEBUG) + cl:Client = None # init @@ -30,7 +35,7 @@ def login() -> int: cl.set_device(DEVICE) sessionSuccess = True if session: - print("[*] Trying logging in with session") + iglog.info("Trying logging in with session") try: cl.set_settings(session) cl.login(ACCOUNT_USERNAME, ACCOUNT_PASSWORD) @@ -42,7 +47,7 @@ def login() -> int: # login with username and password if not sessionSuccess: - print("[*] Trying logging in with username and password") + iglog.info("Trying logging in with username and password") try: old_session = cl.get_settings() cl.set_settings({}) @@ -50,20 +55,20 @@ def login() -> int: cl.login(ACCOUNT_USERNAME, ACCOUNT_PASSWORD) cl.get_timeline_feed() except: - print("[X] Cannot log in") + iglog.error("Cannot log in") return 0 cl.dump_settings("session.json") # return username = cl.account_info().dict()["username"] - print("[V] Logged as %s"%username) + iglog.info("Logged as %s"%username) return 1 # Get account info def account_info() -> dict | None: - print("[*] IG: Fetching account info") + iglog.info("Fetching account info") try: info = cl.account_info().dict() return info @@ -87,11 +92,12 @@ def media_info(code:str) -> dict | None: def upload_media(content:str, paths:List[str]) -> dict | None: try: # uplaod - if len(paths) == 0: return None + if len(paths) == 0: + return None elif len(paths) == 1: media = cl.photo_upload(path=paths[0], caption=content).dict() else: - media = cl.photo_upload(path=paths[0], caption=content).dict() + media = cl.album_upload(paths=paths, caption=content).dict() return media except Exception as e: @@ -108,4 +114,4 @@ def delete_media(code:str) -> int: return 0 except Exception as e: easyExceptionHandler(e) - return 1 \ No newline at end of file + return 1 diff --git a/s3/s3helper.py b/s3/s3helper.py index 18c084c..e8806db 100644 --- a/s3/s3helper.py +++ b/s3/s3helper.py @@ -1,9 +1,14 @@ from typing import Tuple import os import sys +import logging import minio +# logging +s3log = logging.getLogger("s3") +s3log.setLevel(level=logging.INFO) + S3_BUCKET:str = os.getenv("S3_BUCKET") s3 = minio.Minio(endpoint=os.getenv("S3_ENDPOINT").strip(), @@ -12,9 +17,9 @@ s3 = minio.Minio(endpoint=os.getenv("S3_ENDPOINT").strip(), secure=False) # check exist -print("[*] Connecting to Minio") +s3log.info("Connecting to Minio") if not s3.bucket_exists(S3_BUCKET): - print("[X] Where is S3 bucket \"%s\"?"%S3_BUCKET) + s3log.critical("Where is S3 bucket \"%s\"?"%S3_BUCKET) sys.exit(0) diff --git a/utils/tbProcessor.py b/utils/tbProcessor.py index 782a141..dd33d60 100644 --- a/utils/tbProcessor.py +++ b/utils/tbProcessor.py @@ -1,6 +1,7 @@ import json import traceback import os +import logging FILENAME = "./traceback.json" @@ -27,7 +28,8 @@ def debug_info_from_exception(exc) -> dict: } # debug - for s in exc_traceback: print(s) + for s in exc_traceback: + logging.error(s) # must display return debug_info @@ -48,4 +50,4 @@ def easyExceptionHandler(e:Exception): exc_type = type(e).__name__ exc_message = str(e) exc_saved_id = write(e) - print(f"[X] Exception id {exc_saved_id} : {exc_type} : {exc_message}") \ No newline at end of file + logging.error(f"Exception id {exc_saved_id} : {exc_type} : {exc_message}")