fix some bugs

.gitignore

@@ -3,8 +3,5 @@ __pycache__
 session.json
 traceback.json
 test.py
-
 tmp
-tmp/*
+tmp/*
-testFiles
-testFiles/*

Dockerfile

@@ -0,0 +1,18 @@
+FROM python:3.11
+
+WORKDIR /app
+
+# apt install
+RUN apt-get update && \
+    DEBAIN_FRONTEND=noninteractive apt-get install -qy ffmpeg
+# other packages if needed
+# git openssh-server vim net-tools iputils-ping btop tmux wget
+
+# pip3 install
+COPY ./requirements.txt /app/requirements.txt
+COPY ./ffmpeg_python-0.2.0-py3-none-any.whl /app/ffmpeg_python-0.2.0-py3-none-any.whl
+
+RUN pip3 install -r /app/requirements.txt
+RUN pip3 install ffmpeg_python-0.2.0-py3-none-any.whl
+
+EXPOSE 50051

README.md

@@ -0,0 +1,19 @@
+# Niming Backend (IGAPI)
+
+## build
+### with docker-compose
+See ``docker-compose.yml`` (It is a template)
+
+### Manual
+Prepare:
+```
+pip3 install -r requirements.txt
+pip3 install ffmpeg_python-0.2.0-py3-none-any.whl
+```
+
+Run:
+```
+python3 app.py
+```
+
+Shirakami Fubuki is the cutest fox!!!

app.py

@@ -2,6 +2,7 @@ import os
 import sys
 import asyncio
 import threading
+import logging
 
 from sqlalchemy import create_engine
 from instagrapi import Client
@@ -11,16 +12,26 @@ from ig import IG
 from db import dbhelper
 from db.pgclass import Base
 from grpcServer import grpcServer, anoth
-from utils.const import DEBUG
+from utils.const import DEBUG, TMP_DIR
 
 # load_dotenv()
 
+# logging
+logging.basicConfig(
+    level=logging.INFO,
+    format='%(asctime)s | [%(levelname)s] %(name)s - %(message)s'
+)
+
 if DEBUG:
-    print("[*] ===== DEBUG MODE =====")
+    logging.info("===== DEBUG MODE =====")
+    
+# tmp dir
+if not os.path.exists(TMP_DIR):
+    os.mkdir(TMP_DIR)
 
 # Database
 PG_HOST = os.environ.get("PG_HOST", None).strip()
-print("[*] Connecting to Database")
+logging.info("Connecting to Database")
 dbhelper.db = dbhelper.DB(create_engine(PG_HOST))
 Base.metadata.create_all(dbhelper.db._engine)
 

db/dbhelper.py

@@ -146,7 +146,6 @@ def solo_article_set_igid(id:int, igid:str) -> int:
             stmt = update(article_meta).where(article_meta.hash==hash).values(igid=igid)
             session.execute(stmt)
         except Exception as e:
-            print(e)
             err = 1
         session.commit()
     return err

docker-compose.yml

@@ -0,0 +1,26 @@
+version: '3'
+
+# template: docker-compose.yml
+
+services:
+  niming-backend-igrpc:
+    build: .
+    container_name: niming-backend-igrpc
+    volumes:
+      - "igrpc_data/:/app"
+    ports:
+      - "50051:50051"
+      - "10000:10000" # i dont know what is listening this but this is written in docker-compose(dev)
+    environment:
+      - PG_HOST=postgresql+psycopg2://root:password@ip:port/niming_db
+      - ACCOUNT_USERNAME=
+      - ACCOUNT_PASSWORD=
+      - S3_ENDPOINT=ip:port
+      - S3_ACCESS_KEY=
+      - S3_SECRET_KEY=
+      - S3_BUCKET=nmfs
+    restart: unless-stopped
+    working_dir: /app
+    command: python3 /app/app.py
+    #networks:
+    #  - networkName

grpcServer/anoth.py

@@ -1,22 +1,29 @@
 import time
 import random
+import logging
 
 from grpcServer import postProcessor
 from utils.ThreadSafeOrderedDict import ThreadSafeOrderedDict
 from utils.const import ANOTH_INTERVAL_MIN, ANOTH_INTERVAL_MAX
+from utils.tbProcessor import easyExceptionHandler
 from db import dbhelper
 
+# logging
+anothlog = logging.getLogger("anoth")
+anothlog.setLevel(level=logging.INFO)
+
+# task queue
 task = ThreadSafeOrderedDict()
 
 def task_round():
     t = task.popitem(last=False)
     if not t: # 沒任務
-        print("[*] No task in queue")
+        anothlog.info("No task in queue")
         return
     
     aid = t[1]["aid"]
     type = t[0].split("-")[0]
-    print("[*] Task %s(target_aid=%d)"%(type, aid))
+    anothlog.info("Task %s(target_aid=%d)"%(type, aid))
     
     if type == "upload": # upload
         msg, err = postProcessor.upload(aid)
@@ -27,21 +34,24 @@ def task_round():
         msg, err = "Invalid task type %s"%type, 1
         
     if err:
-        print("[X] Task failed: %s"%msg)
+        anothlog.error("Task failed: %s"%msg)
     elif type == "upload":
         dberr = dbhelper.solo_article_set_igid(id=aid, igid=msg)
         if dberr:
-            print("[X] Task %s(target_aid=%d): Set igid failed"%(type, aid))
+            anothlog.error("Task %s(target_aid=%d): Set igid failed"%(type, aid))
         
-    print("[*] Task Done")
+    anothlog.info("Task Done")
     return
 
 
 def run():
-    print("[*] Upload/Delete Processor Started")
+    anothlog.info("Upload/Delete Processor Started")
     while True:
-        task_round()
+        try:
+            task_round()
+        except Exception as e:
+            easyExceptionHandler(e)
         
         sleep = random.randint(ANOTH_INTERVAL_MIN, ANOTH_INTERVAL_MAX)
-        print("[*] Next Round After %ds"%sleep)
+        anothlog.info("Next Round After %ds"%sleep)
         time.sleep(sleep)

grpcServer/grpcServer.py

@@ -1,4 +1,5 @@
 # from concurrent import futures
+import logging
 
 import grpc
 from cachetools import cached, TTLCache
@@ -10,6 +11,10 @@ from grpcServer import anoth
 from grpcServer.protobuf import igapi_pb2_grpc
 from grpcServer.protobuf.igapi_pb2 import Request, Reply
 
+# logging
+grpclog = logging.getLogger("grpc")
+grpclog.setLevel(level=logging.INFO)
+
 # call account info  / login
 cache_accinfo = TTLCache(maxsize=1, ttl=GRPC_ACCINFO_CACHE)
 @cached(cache_accinfo)
@@ -28,7 +33,7 @@ def call_IG_login():
 # 考慮一下如果同時發起多的請求，asyncio可能會搞到被ban號(IG)
 class IGAPI_Server(igapi_pb2_grpc.IGAPIServicer):
     async def account_info(self, request: Request, context) -> Reply:
-        print("[*] Request: account_info")
+        grpclog.info("Request: account_info")
         account = call_IG_account_info()
         if account:
             result = {
@@ -42,9 +47,9 @@ class IGAPI_Server(igapi_pb2_grpc.IGAPIServicer):
 
     
     async def login(self, request: Request, context) -> Reply:
-        print("[*] Request: login")
+        grpclog.info("Request: login")
         if len(cache_login): # cache has not expired
-            print("[*] Login: Cooldown")
+            grpclog.info("Login: Cooldown")
             return Reply(err=1, result={"error":"Cooldown"})
         else:
             login = call_IG_login()
@@ -55,7 +60,7 @@ class IGAPI_Server(igapi_pb2_grpc.IGAPIServicer):
     
     
     async def upload(self, request: Request, context) -> Reply:
-        print("[*] Request: upload")
+        grpclog.info("Request: upload")
         aid = request.code
         
         # 檢查 - 可見
@@ -83,7 +88,7 @@ class IGAPI_Server(igapi_pb2_grpc.IGAPIServicer):
     
     
     async def delete(self, request: Request, context) -> Reply:
-        print("[*] Request: delete")
+        grpclog.info("Request: delete")
         # article id
         aid = request.code
         # igid from args
@@ -108,7 +113,7 @@ class IGAPI_Server(igapi_pb2_grpc.IGAPIServicer):
     
     
     async def queue(self, request:Request, context) -> Reply:
-        print("[*] Request: queue")
+        grpclog.info("Request: queue")
         t = anoth.task.items()
         reply = { _[0]:str(_[1]["aid"]) for _ in t }
         return Reply(err=0, result=reply)
@@ -116,7 +121,7 @@ class IGAPI_Server(igapi_pb2_grpc.IGAPIServicer):
     
     async def setting(self, request:Request, context) -> Reply:
         # not done
-        print("[*] Request: setting")
+        grpclog.info("Request: setting")
         return Reply(err=1, result={"error":"Not Done"})
 
 
@@ -128,5 +133,5 @@ async def serve() -> None:
     )
     server.add_insecure_port("[::]:50051")
     await server.start()
-    print("[*] gRPC Server listening on 0.0.0.0:50051")
+    grpclog.info("gRPC Server listening on 0.0.0.0:50051")
     await server.wait_for_termination()

ig/IG.py

@@ -1,4 +1,5 @@
 import os
+import logging
 from typing import List
 
 from instagrapi import Client
@@ -6,6 +7,10 @@ from instagrapi import Client
 from utils.tbProcessor import easyExceptionHandler
 from utils.const import DEVICE
 
+# logging
+iglog = logging.getLogger("ig")
+iglog.setLevel(level=logging.DEBUG)
+
 cl:Client = None
 
 # init
@@ -30,7 +35,7 @@ def login() -> int:
     cl.set_device(DEVICE)
     sessionSuccess = True
     if session:
-        print("[*] Trying logging in with session")
+        iglog.info("Trying logging in with session")
         try:
             cl.set_settings(session)
             cl.login(ACCOUNT_USERNAME, ACCOUNT_PASSWORD)
@@ -42,7 +47,7 @@ def login() -> int:
 
     # login with username and password
     if not sessionSuccess:
-        print("[*] Trying logging in with username and password")
+        iglog.info("Trying logging in with username and password")
         try:
             old_session = cl.get_settings()
             cl.set_settings({})
@@ -50,20 +55,20 @@ def login() -> int:
             cl.login(ACCOUNT_USERNAME, ACCOUNT_PASSWORD)
             cl.get_timeline_feed()
         except:
-            print("[X] Cannot log in")
+            iglog.error("Cannot log in")
             return 0
     
     cl.dump_settings("session.json")
     
     # return
     username = cl.account_info().dict()["username"]
-    print("[V] Logged as %s"%username)
+    iglog.info("Logged as %s"%username)
     return 1
 
 
 # Get account info
 def account_info() -> dict | None:
-    print("[*] IG: Fetching account info")
+    iglog.info("Fetching account info")
     try:
         info = cl.account_info().dict()
         return info
@@ -87,11 +92,12 @@ def media_info(code:str) -> dict | None:
 def upload_media(content:str, paths:List[str]) -> dict | None:
     try:
         # uplaod
-        if len(paths) == 0:  return None
+        if len(paths) == 0:
+            return None
         elif len(paths) == 1:
             media = cl.photo_upload(path=paths[0], caption=content).dict()
         else:
-            media = cl.photo_upload(path=paths[0], caption=content).dict()
+            media = cl.album_upload(paths=paths, caption=content).dict()
             
         return media
     except Exception as e:
@@ -108,4 +114,4 @@ def delete_media(code:str) -> int:
         return 0
     except Exception as e:
         easyExceptionHandler(e)
-        return 1
+        return 1

s3/s3helper.py

@@ -1,9 +1,14 @@
 from typing import Tuple
 import os
 import sys
+import logging
 
 import minio
 
+# logging
+s3log = logging.getLogger("s3")
+s3log.setLevel(level=logging.INFO)
+
 S3_BUCKET:str = os.getenv("S3_BUCKET")
 
 s3 = minio.Minio(endpoint=os.getenv("S3_ENDPOINT").strip(),
@@ -12,9 +17,9 @@ s3 = minio.Minio(endpoint=os.getenv("S3_ENDPOINT").strip(),
                  secure=False)
 
 # check exist
-print("[*] Connecting to Minio")
+s3log.info("Connecting to Minio")
 if not s3.bucket_exists(S3_BUCKET):
-    print("[X] Where is S3 bucket \"%s\"?"%S3_BUCKET)
+    s3log.critical("Where is S3 bucket \"%s\"?"%S3_BUCKET)
     sys.exit(0)
 
 

utils/tbProcessor.py

@@ -1,6 +1,7 @@
 import json
 import traceback
 import os
+import logging
 
 FILENAME = "./traceback.json"
 
@@ -27,7 +28,8 @@ def debug_info_from_exception(exc) -> dict:
     }
     
     # debug
-    for s in exc_traceback: print(s)
+    for s in exc_traceback:
+        logging.error(s) # must display
 
     return debug_info
 
@@ -48,4 +50,4 @@ def easyExceptionHandler(e:Exception):
     exc_type = type(e).__name__
     exc_message = str(e)
     exc_saved_id = write(e)
-    print(f"[X] Exception id {exc_saved_id} : {exc_type} : {exc_message}")
+    logging.error(f"Exception id {exc_saved_id} : {exc_type} : {exc_message}")